From pypy.commits at gmail.com  Wed Feb  1 02:47:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 23:47:51 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove the deprecated (and not working)
 feature llop.call_at_startup, 
Message-ID: <589192a7.cfc41c0a.34046.97d4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89864:6fdd39fa2ae8
Date: 2017-02-01 08:47 +0100
http://bitbucket.org/pypy/pypy/changeset/6fdd39fa2ae8/

Log:	Remove the deprecated (and not working) feature
	llop.call_at_startup, which I forgot in the rpython-hash branch. Fix
	test.

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,7 +539,6 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
-    'call_at_startup':      LLOp(canrun=True),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -742,9 +742,6 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
-def op_call_at_startup(init_func):
-    pass    # do nothing
-
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -60,7 +60,6 @@
         self.completed = False
 
         self.instrument_ncounter = 0
-        self.call_at_startup = set()
 
     def gettypedefnode(self, T, varlength=None):
         if varlength is None:
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -941,18 +941,3 @@
                 cdecl(typename, ''),
                 self.expr(op.args[0]),
                 self.expr(op.result))
-
-    def OP_CALL_AT_STARTUP(self, op):
-        c = op.args[0]
-        if not isinstance(c, Constant):
-            # Bah, maybe it comes from a same_as(const) just before...
-            # Can occur if running without backendopts
-            for op1 in self._current_block.operations:
-                if op1.result is op.args[0]:
-                    assert op1.opname == "same_as"
-                    c = op1.args[0]
-                    break
-            assert isinstance(c, Constant)
-        func = self.expr(c)
-        self.db.call_at_startup.add(func)
-        return '/* call_at_startup %s */' % (func,)
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1063,23 +1063,39 @@
         assert out.strip() == expected
 
     def test_call_at_startup(self):
-        from rpython.rtyper.lltypesystem import lltype
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rtyper.annlowlevel import llhelper
+        from rpython.rtyper.extregistry import ExtRegistryEntry
+
         class State:
             seen = 0
         state = State()
         def startup():
             state.seen += 1
-        F = lltype.Ptr(lltype.FuncType([], lltype.Void))
+        def enablestartup():
+            "NOT_RPYTHON"
         def entry_point(argv):
             state.seen += 100
             assert state.seen == 101
             print 'ok'
-            ll = llhelper(F, startup)
-            llop.call_at_startup(lltype.Void, ll)
+            enablestartup()
             return 0
 
+        class Entry(ExtRegistryEntry):
+            _about_ = enablestartup
+
+            def compute_result_annotation(self):
+                bk = self.bookkeeper
+                s_callable = bk.immutablevalue(startup)
+                key = (enablestartup,)
+                bk.emulate_pbc_call(key, s_callable, [])
+
+            def specialize_call(self, hop):
+                hop.exception_cannot_occur()
+                bk = hop.rtyper.annotator.bookkeeper
+                s_callable = bk.immutablevalue(startup)
+                r_callable = hop.rtyper.getrepr(s_callable)
+                ll_init = r_callable.get_unique_llfn().value
+                bk.annotator.translator._call_at_startup.append(ll_init)
+
         t, cbuilder = self.compile(entry_point)
         out = cbuilder.cmdexec('')
         assert out.strip() == 'ok'

From pypy.commits at gmail.com  Wed Feb  1 02:49:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 31 Jan 2017 23:49:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5891931d.2a99df0a.43838.3add@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89865:6208408f7077
Date: 2017-02-01 08:49 +0100
http://bitbucket.org/pypy/pypy/changeset/6208408f7077/

Log:	hg merge default

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,7 +539,6 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
-    'call_at_startup':      LLOp(canrun=True),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -742,9 +742,6 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
-def op_call_at_startup(init_func):
-    pass    # do nothing
-
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -60,7 +60,6 @@
         self.completed = False
 
         self.instrument_ncounter = 0
-        self.call_at_startup = set()
 
     def gettypedefnode(self, T, varlength=None):
         if varlength is None:
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -941,18 +941,3 @@
                 cdecl(typename, ''),
                 self.expr(op.args[0]),
                 self.expr(op.result))
-
-    def OP_CALL_AT_STARTUP(self, op):
-        c = op.args[0]
-        if not isinstance(c, Constant):
-            # Bah, maybe it comes from a same_as(const) just before...
-            # Can occur if running without backendopts
-            for op1 in self._current_block.operations:
-                if op1.result is op.args[0]:
-                    assert op1.opname == "same_as"
-                    c = op1.args[0]
-                    break
-            assert isinstance(c, Constant)
-        func = self.expr(c)
-        self.db.call_at_startup.add(func)
-        return '/* call_at_startup %s */' % (func,)
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1063,23 +1063,39 @@
         assert out.strip() == expected
 
     def test_call_at_startup(self):
-        from rpython.rtyper.lltypesystem import lltype
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rtyper.annlowlevel import llhelper
+        from rpython.rtyper.extregistry import ExtRegistryEntry
+
         class State:
             seen = 0
         state = State()
         def startup():
             state.seen += 1
-        F = lltype.Ptr(lltype.FuncType([], lltype.Void))
+        def enablestartup():
+            "NOT_RPYTHON"
         def entry_point(argv):
             state.seen += 100
             assert state.seen == 101
             print 'ok'
-            ll = llhelper(F, startup)
-            llop.call_at_startup(lltype.Void, ll)
+            enablestartup()
             return 0
 
+        class Entry(ExtRegistryEntry):
+            _about_ = enablestartup
+
+            def compute_result_annotation(self):
+                bk = self.bookkeeper
+                s_callable = bk.immutablevalue(startup)
+                key = (enablestartup,)
+                bk.emulate_pbc_call(key, s_callable, [])
+
+            def specialize_call(self, hop):
+                hop.exception_cannot_occur()
+                bk = hop.rtyper.annotator.bookkeeper
+                s_callable = bk.immutablevalue(startup)
+                r_callable = hop.rtyper.getrepr(s_callable)
+                ll_init = r_callable.get_unique_llfn().value
+                bk.annotator.translator._call_at_startup.append(ll_init)
+
         t, cbuilder = self.compile(entry_point)
         out = cbuilder.cmdexec('')
         assert out.strip() == 'ok'

From pypy.commits at gmail.com  Wed Feb  1 03:09:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 00:09:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Reduce the diff with CPython 3.5's
 test_hash.py
Message-ID: <589197cc.ec98df0a.9eaae.3bdd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89866:314a31846fa6
Date: 2017-02-01 09:09 +0100
http://bitbucket.org/pypy/pypy/changeset/314a31846fa6/

Log:	Reduce the diff with CPython 3.5's test_hash.py

diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -43,7 +43,7 @@
 
 def skip_unless_internalhash(test):
     """Skip decorator for tests that depend on SipHash24 or FNV"""
-    ok = sys.hash_info.algorithm in {"fnv", "siphash24", "rpython"}
+    ok = sys.hash_info.algorithm in {"fnv", "siphash24"}
     msg = "Requires SipHash24 or FNV"
     return test if ok else unittest.skip(msg)(test)
 
@@ -189,7 +189,6 @@
         stdout = out[1].strip()
         return int(stdout)
 
-    @impl_detail("PyPy does not support hash randomization", pypy=False)
     def test_randomized_hash(self):
         # two runs should return different hashes
         run1 = self.get_hash(self.repr_, seed='random')
@@ -197,10 +196,6 @@
         self.assertNotEqual(run1, run2)
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
-    if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -2
-    else:
-        EMPTY_STRING_HASH = 0
     repr_ = None
     repr_long = None
 
@@ -242,9 +237,9 @@
             [-283066365, -4576729883824601543, -271871407,
              -3927695501187247084],
         ],
-        'rpython': [
-            # This is for PyPy.  NOTE: PyPy does not support hash
-            # randomization for now, so the results don't depend on the seed.
+        'fnv-pypy': [
+            # This is for PyPy, whose fnv algorithm does not support hash
+            # randomization, so the results don't depend on the seed.
             # seed 0, 'abc'
             [-1600925533, 1453079729188098211, -1600925533,
              1453079729188098211],
@@ -273,6 +268,8 @@
         else:
             assert(sys.byteorder == 'big')
             platform = 3 if IS_64BIT else 2
+        if algorithm == 'fnv' and check_impl_detail(pypy=True):
+            algorithm = 'fnv-pypy'
         return self.known_hashes[algorithm][position][platform]
 
     def test_null_hash(self):
@@ -280,8 +277,7 @@
         known_hash_of_obj = self.get_expected_hash(0, 3)
 
         # Randomization is enabled by default:
-        if check_impl_detail(pypy=False):
-            self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
+        self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
 
         # It can also be disabled by setting the seed to 0:
         self.assertEqual(self.get_hash(self.repr_, seed=0), known_hash_of_obj)
@@ -307,7 +303,7 @@
     repr_long = repr('abcdefghijk')
     repr_ucs2 = repr('&#228;&#250;&#8721;&#8455;')
 
-    @impl_detail("hash('') == -2 on PyPy", pypy=False)
+    @impl_detail("hash('') != 0 on PyPy", pypy=False)
     @skip_unless_internalhash
     def test_empty_string(self):
         self.assertEqual(hash(""), 0)
@@ -324,18 +320,20 @@
     repr_ = repr(b'abc')
     repr_long = repr(b'abcdefghijk')
 
+    @impl_detail("hash('') != 0 on PyPy", pypy=False)
     @skip_unless_internalhash
     def test_empty_string(self):
-        self.assertEqual(hash(b""), self.EMPTY_STRING_HASH)
+        self.assertEqual(hash(b""), 0)
 
 class MemoryviewHashRandomizationTests(StringlikeHashRandomizationTests,
                                        unittest.TestCase):
     repr_ = "memoryview(b'abc')"
     repr_long = "memoryview(b'abcdefghijk')"
 
+    @impl_detail("hash('') != 0 on PyPy", pypy=False)
     @skip_unless_internalhash
     def test_empty_string(self):
-        self.assertEqual(hash(memoryview(b"")), self.EMPTY_STRING_HASH)
+        self.assertEqual(hash(memoryview(b"")), 0)
 
 class DatetimeTests(HashRandomizationTests):
     def get_hash_command(self, repr_):

From pypy.commits at gmail.com  Wed Feb  1 03:29:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 00:29:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix datetime.time.__hash__ to avoid
 (sometimes) giving a result computed
Message-ID: <58919c61.2a99df0a.43838.4a0d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89867:1e117d8ffc90
Date: 2017-02-01 09:28 +0100
http://bitbucket.org/pypy/pypy/changeset/1e117d8ffc90/

Log:	Fix datetime.time.__hash__ to avoid (sometimes) giving a result
	computed without hashing a string, and thus non-random

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -706,7 +706,7 @@
 
     @classmethod
     def fromordinal(cls, n):
-        """Contruct a date from a proleptic Gregorian ordinal.
+        """Construct a date from a proleptic Gregorian ordinal.
 
         January 1 of year 1 is day 1.  Only the year, month and day are
         non-zero in the result.
@@ -1149,14 +1149,14 @@
             if not tzoff:  # zero or None
                 self._hashcode = hash(self._getstate()[0])
             else:
-                h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff,
-                              timedelta(hours=1))
-                assert not m % timedelta(minutes=1), "whole minute"
-                m //= timedelta(minutes=1)
-                if 0 <= h < 24:
-                    self._hashcode = hash(time(h, m, self.second, self.microsecond))
-                else:
-                    self._hashcode = hash((h, m, self.second, self.microsecond))
+                # PyPy: uses the same algo as _datetimemodule.c, which
+                # unlike the pure Python version always relies on the
+                # nondeterministic hash on strings
+                seconds = self.hour * 3600 + self.minute * 60 + self.second
+                temp1 = timedelta(seconds=self.seconds,
+                                  microseconds=self.microseconds)
+                temp2 = temp1 - tzoff
+                self._hashcode = hash(temp2)
         return self._hashcode
 
     # Conversion to string

From pypy.commits at gmail.com  Wed Feb  1 03:33:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 00:33:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip test on runappdirect
Message-ID: <58919d59.cc881c0a.7c41.a5d4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89868:d5619ea4fcdb
Date: 2017-02-01 09:32 +0100
http://bitbucket.org/pypy/pypy/changeset/d5619ea4fcdb/

Log:	skip test on runappdirect

diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -98,6 +98,9 @@
 
 class AppTestBytesObject:
 
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
     def test_constructor(self):
         assert bytes() == b''
         assert bytes(3) == b'\0\0\0'
@@ -789,6 +792,8 @@
         raises(LookupError, 'hello'.encode, 'base64')
 
     def test_hash(self):
+        if self.runappdirect:
+            skip("randomized hash by default")
         # check that we have the same hash as CPython for at least 31 bits
         # (but don't go checking CPython's special case -1)
         # disabled: assert hash('') == 0 --- different special case

From pypy.commits at gmail.com  Wed Feb  1 03:48:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 00:48:11 -0800 (PST)
Subject: [pypy-commit] pypy default: Check that computing hashes with
 siphash24 can raise MemoryError but nothing more
Message-ID: <5891a0cb.d5941c0a.52d98.a9ed@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89869:70e009029493
Date: 2017-02-01 09:45 +0100
http://bitbucket.org/pypy/pypy/changeset/70e009029493/

Log:	Check that computing hashes with siphash24 can raise MemoryError but
	nothing more

diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -281,6 +281,8 @@
 
 def test_elidable_kinds():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
+    from rpython.rlib.objectmodel import compute_hash
+    from rpython.rlib.rsiphash import enable_siphash24
 
     @jit.elidable
     def f1(n, m):
@@ -293,12 +295,17 @@
         if n > m:
             raise ValueError
         return n + m
+    @jit.elidable
+    def f4(n, m):
+        return compute_hash(str(n) + str(m))
 
     def f(n, m):
         a = f1(n, m)
         b = f2(n, m)
         c = f3(n, m)
-        return a + len(b) + c
+        d = f4(n, m)
+        enable_siphash24()
+        return a + len(b) + c + d
 
     rtyper = support.annotate(f, [7, 9])
     jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
@@ -309,7 +316,8 @@
     for index, expected in [
             (0, EffectInfo.EF_ELIDABLE_CANNOT_RAISE),
             (1, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR),
-            (2, EffectInfo.EF_ELIDABLE_CAN_RAISE)]:
+            (2, EffectInfo.EF_ELIDABLE_CAN_RAISE),
+            (3, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR)]:
         call_op = f_graph.startblock.operations[index]
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)

From pypy.commits at gmail.com  Wed Feb  1 03:48:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 00:48:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5891a0cd.eb88df0a.0977.4d29@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89870:4e608fcc30a0
Date: 2017-02-01 09:47 +0100
http://bitbucket.org/pypy/pypy/changeset/4e608fcc30a0/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -71,7 +71,10 @@
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
             i99 = unicodehash(p10)
-            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_unicodePtr), p10, descr=<Calli . r EF=2>)
+            # NOTE: with siphash24, notably on unicodes, computing the hash
+            # may raise MemoryError
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_unicodePtr), p10, descr=<Calli . r EF=5>)
+            guard_no_exception(descr=...)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{

From pypy.commits at gmail.com  Wed Feb  1 04:06:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 01:06:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5891a509.d4b81c0a.8a3a6.15c0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89871:5cac6275c51a
Date: 2017-02-01 10:05 +0100
http://bitbucket.org/pypy/pypy/changeset/5cac6275c51a/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_array.py b/pypy/module/pypyjit/test_pypy_c/test_array.py
--- a/pypy/module/pypyjit/test_pypy_c/test_array.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_array.py
@@ -28,7 +28,7 @@
     def test_array_sum(self):
         def main():
             from array import array
-            img = array("i", range(128) * 5) * 480
+            img = array("i", list(range(128)) * 5) * 480
             l, i = 0, 0
             while i < len(img):
                 l += img[i]

From pypy.commits at gmail.com  Wed Feb  1 04:06:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 01:06:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: make the JIT see through tup[i:j]
Message-ID: <5891a50b.a1aedf0a.ff632.55b4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89872:2c9ea4e91032
Date: 2017-02-01 10:05 +0100
http://bitbucket.org/pypy/pypy/changeset/2c9ea4e91032/

Log:	make the JIT see through tup[i:j]

diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -196,12 +196,20 @@
         items = self.tolist()
         length = len(items)
         start, stop, step, slicelength = w_index.indices4(space, length)
+        if step == 1:
+            subitems = items[start:stop]
+        else:
+            subitems = self._getslice_advanced(items, start, step, slicelength)
+        return space.newtuple(subitems)
+
+    @staticmethod
+    def _getslice_advanced(items, start, step, slicelength):
         assert slicelength >= 0
         subitems = [None] * slicelength
         for i in range(slicelength):
             subitems[i] = items[start]
             start += step
-        return space.newtuple(subitems)
+        return subitems
 
     def descr_getnewargs(self, space):
         return space.newtuple([space.newtuple(self.tolist())])

From pypy.commits at gmail.com  Wed Feb  1 04:13:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 01:13:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: update docstring
Message-ID: <5891a6ab.ccaddf0a.91179.4944@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89873:87a4dd850dad
Date: 2017-02-01 10:12 +0100
http://bitbucket.org/pypy/pypy/changeset/87a4dd850dad/

Log:	update docstring

diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1008,7 +1008,8 @@
 
 @cpython_api([rffi.INT_real, CWCHARPP], lltype.Void)
 def PySys_SetArgv(space, argc, argv):
-    """This function works like PySys_SetArgvEx() with updatepath set to 1."""
+    """This function works like PySys_SetArgvEx() with updatepath set
+    to 1 unless the python interpreter was started with the option -I."""
     raise NotImplementedError
 
 @cpython_api([rffi.CWCHARP], lltype.Void)

From pypy.commits at gmail.com  Wed Feb  1 04:13:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 01:13:25 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: the -I option is already done
Message-ID: <5891a6b5.09bb1c0a.66c7a.b6cf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5769:9f0ab06928bb
Date: 2017-02-01 10:13 +0100
http://bitbucket.org/pypy/extradoc/changeset/9f0ab06928bb/

Log:	the -I option is already done

diff --git a/planning/py3.5/milestone-2-progress.rst b/planning/py3.5/milestone-2-progress.rst
--- a/planning/py3.5/milestone-2-progress.rst
+++ b/planning/py3.5/milestone-2-progress.rst
@@ -70,6 +70,7 @@
   [DONE, rpython-hash branch]
 
 * New command line option for isolated mode.
+  [I think it is already DONE]
 
 * Enhancements to multiprocessing modules.
 

From pypy.commits at gmail.com  Wed Feb  1 04:14:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 01:14:23 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: nothing to do here
Message-ID: <5891a6ef.da6d1c0a.98692.c429@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5770:61788bb1879c
Date: 2017-02-01 10:14 +0100
http://bitbucket.org/pypy/extradoc/changeset/61788bb1879c/

Log:	nothing to do here

diff --git a/planning/py3.5/milestone-2-progress.rst b/planning/py3.5/milestone-2-progress.rst
--- a/planning/py3.5/milestone-2-progress.rst
+++ b/planning/py3.5/milestone-2-progress.rst
@@ -75,6 +75,7 @@
 * Enhancements to multiprocessing modules.
 
 * HTTP cookie parsing is now stricter (issue 22796).
+  [PURELY APP-LEVEL]
 
 The measure of when this milestone is reached is based on the
 following criteria: we can take a number of C extension modules that

From pypy.commits at gmail.com  Wed Feb  1 04:52:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 01:52:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: translation fix
Message-ID: <5891aff4.d3811c0a.9f514.be98@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89874:5699b5075b86
Date: 2017-02-01 10:52 +0100
http://bitbucket.org/pypy/pypy/changeset/5699b5075b86/

Log:	translation fix

diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -196,7 +196,10 @@
         items = self.tolist()
         length = len(items)
         start, stop, step, slicelength = w_index.indices4(space, length)
-        if step == 1:
+        if slicelength == 0:
+            subitems = []
+        elif step == 1:
+            assert 0 <= start <= stop
             subitems = items[start:stop]
         else:
             subitems = self._getslice_advanced(items, start, step, slicelength)

From pypy.commits at gmail.com  Wed Feb  1 05:00:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 02:00:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-multiprocessing: close abandoned branch
Message-ID: <5891b1da.0f96df0a.7e973.663c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-multiprocessing
Changeset: r89875:72905ffbd8b5
Date: 2017-02-01 10:59 +0100
http://bitbucket.org/pypy/pypy/changeset/72905ffbd8b5/

Log:	close abandoned branch


From pypy.commits at gmail.com  Wed Feb  1 07:44:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 04:44:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip this test
Message-ID: <5891d820.2da9df0a.7f654.a79f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89876:8af33396f2f2
Date: 2017-02-01 13:43 +0100
http://bitbucket.org/pypy/pypy/changeset/8af33396f2f2/

Log:	skip this test

diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -308,6 +308,7 @@
     def test_empty_string(self):
         self.assertEqual(hash(""), 0)
 
+    @impl_detail("hash(ucs2) differs on PyPy if unichar is 4 bytes", pypy=False)
     @skip_unless_internalhash
     def test_ucs2_string(self):
         h = self.get_expected_hash(3, 6)

From pypy.commits at gmail.com  Wed Feb  1 07:52:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 04:52:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix for 1e117d8ffc90
Message-ID: <5891d9f1.cf0d1c0a.8fab2.16b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89877:c700ecba3a07
Date: 2017-02-01 13:51 +0100
http://bitbucket.org/pypy/pypy/changeset/c700ecba3a07/

Log:	fix for 1e117d8ffc90

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -1145,18 +1145,17 @@
     def __hash__(self):
         """Hash."""
         if self._hashcode == -1:
+            # PyPy: uses the same algo as _datetimemodule.c, which
+            # unlike the pure Python version always relies on the
+            # nondeterministic hash on strings
+            temp1 = timedelta(hours=self._hour,
+                              minutes=self._minute,
+                              seconds=self._second,
+                              microseconds=self._microsecond)
             tzoff = self.utcoffset()
-            if not tzoff:  # zero or None
-                self._hashcode = hash(self._getstate()[0])
-            else:
-                # PyPy: uses the same algo as _datetimemodule.c, which
-                # unlike the pure Python version always relies on the
-                # nondeterministic hash on strings
-                seconds = self.hour * 3600 + self.minute * 60 + self.second
-                temp1 = timedelta(seconds=self.seconds,
-                                  microseconds=self.microseconds)
-                temp2 = temp1 - tzoff
-                self._hashcode = hash(temp2)
+            if tzoff:  # not zero, not None
+                temp1 -= tzoff
+            self._hashcode = hash(temp1)
         return self._hashcode
 
     # Conversion to string

From pypy.commits at gmail.com  Wed Feb  1 08:28:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 05:28:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test and fix
Message-ID: <5891e28f.958c1c0a.73dde.3658@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89879:9dc20d26cd82
Date: 2017-02-01 14:27 +0100
http://bitbucket.org/pypy/pypy/changeset/9dc20d26cd82/

Log:	test and fix

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -216,9 +216,10 @@
 
     def semaphore_unlink(space, w_name):
         name = space.str_w(w_name)
-        res = _sem_unlink(name)
-        if res < 0:
-            raise oefmt(space.w_OSError, "sem unlink failed with errno: %d", rposix.get_saved_errno())
+        try:
+            sem_unlink(name)
+        except OSError as e:
+            raise wrap_oserror(space, e)
 
 class CounterState:
     def __init__(self, space):
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -22,10 +22,13 @@
     @py.test.mark.skipif("sys.platform == 'win32'")
     def test_sem_unlink(self):
         from _multiprocessing import sem_unlink
+        import errno
         try:
             sem_unlink("non-existent")
-        except OSError:
-            pass
+        except OSError as e:
+            assert e.errno in (errno.ENOENT, errno.EINVAL)
+        else:
+            assert 0, "should have raised"
 
     def test_semaphore(self):
         from _multiprocessing import SemLock

From pypy.commits at gmail.com  Wed Feb  1 08:28:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 05:28:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5891e28d.0a081c0a.f5bd4.92b9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89878:f3679e41dc3f
Date: 2017-02-01 13:56 +0100
http://bitbucket.org/pypy/pypy/changeset/f3679e41dc3f/

Log:	fix test

diff --git a/lib-python/3/test/_test_multiprocessing.py b/lib-python/3/test/_test_multiprocessing.py
--- a/lib-python/3/test/_test_multiprocessing.py
+++ b/lib-python/3/test/_test_multiprocessing.py
@@ -2893,12 +2893,14 @@
         a = Foo()
         util.Finalize(a, conn.send, args=('a',))
         del a           # triggers callback for a
+        import gc; gc.collect()
 
         b = Foo()
         close_b = util.Finalize(b, conn.send, args=('b',))
         close_b()       # triggers callback for b
         close_b()       # does nothing because callback has already been called
         del b           # does nothing because callback has already been called
+        import gc; gc.collect()
 
         c = Foo()
         util.Finalize(c, conn.send, args=('c',))

From pypy.commits at gmail.com  Wed Feb  1 08:37:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 05:37:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: _thread.error is RuntimeError, now
Message-ID: <5891e48a.d18bdf0a.2b2da.d7b7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89880:20483147e347
Date: 2017-02-01 14:36 +0100
http://bitbucket.org/pypy/pypy/changeset/20483147e347/

Log:	_thread.error is RuntimeError, now

diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py
--- a/pypy/module/thread/__init__.py
+++ b/pypy/module/thread/__init__.py
@@ -24,7 +24,7 @@
         '_set_sentinel':          'os_lock._set_sentinel',
         '_local':                 'os_local.Local',
         'TIMEOUT_MAX':            'space.wrap(float(os_lock.TIMEOUT_MAX // 1000000))',
-        'error':                  'space.fromcache(error.Cache).w_error',
+        'error':                  'space.w_RuntimeError',
     }
 
     def __init__(self, space, *args):
diff --git a/pypy/module/thread/error.py b/pypy/module/thread/error.py
--- a/pypy/module/thread/error.py
+++ b/pypy/module/thread/error.py
@@ -1,9 +1,5 @@
 from pypy.interpreter.error import OperationError
 
-class Cache:
-    def __init__(self, space):
-        self.w_error = space.new_exception_class("thread.error")
-
 def wrap_thread_error(space, msg):
-    w_error = space.fromcache(Cache).w_error
+    w_error = space.w_RuntimeError    # since CPython 3.3
     return OperationError(w_error, space.wrap(msg))
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -34,6 +34,10 @@
             time.sleep(0.2)
             remaining = thread._count()
 
+    def test_thread_error(self):
+        import _thread
+        assert _thread.error is RuntimeError   # uh, since CPython 3.3
+
     def test_start_new_thread(self):
         import _thread
         feedback = []

From pypy.commits at gmail.com  Wed Feb  1 09:06:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 06:06:31 -0800 (PST)
Subject: [pypy-commit] pypy default: uh?
Message-ID: <5891eb67.0d1a1c0a.6eb67.29d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89881:22cb2a83ab9f
Date: 2017-02-01 14:56 +0100
http://bitbucket.org/pypy/pypy/changeset/22cb2a83ab9f/

Log:	uh?

diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -95,8 +95,8 @@
     def test_write_code_builtin_forbidden(self):
         def f(*args):
             return 42
-            raises(TypeError, "dir.func_code = f.func_code")
-            raises(TypeError, "list.append.im_func.func_code = f.func_code")
+        raises(TypeError, "dir.func_code = f.func_code")
+        raises(TypeError, "list.append.im_func.func_code = f.func_code")
 
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")

From pypy.commits at gmail.com  Wed Feb  1 09:09:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 06:09:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5891ec2d.b296df0a.eab4e.d25d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89883:ae853aa08f40
Date: 2017-02-01 15:09 +0100
http://bitbucket.org/pypy/pypy/changeset/ae853aa08f40/

Log:	hg merge default

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -207,10 +207,16 @@
 
     def getdict(self, space):
         if self.w_func_dict is None:
+            if not self.can_change_code:
+                raise oefmt(space.w_AttributeError,
+                            "cannot set extra attributes on built-in functions")
             self.w_func_dict = space.newdict(instance=True)
         return self.w_func_dict
 
     def setdict(self, space, w_dict):
+        if not self.can_change_code:
+            raise oefmt(space.w_AttributeError,
+                        "cannot set __dict__ on built-in functions")
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError,
                         "setting function's dictionary to a non-dict")
@@ -695,7 +701,7 @@
                           func.defs_w, None, func.closure,
                           None, func.name)
         self.w_doc = func.w_doc
-        self.w_func_dict = func.w_func_dict
+        #self.w_func_dict = func.w_func_dict---nowadays, always None
         self.w_module = func.w_module
         self.w_kw_defs = func.w_kw_defs
 
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -160,10 +160,14 @@
     def test_write_code_builtin_forbidden(self):
         def f(*args):
             return 42
-        if hasattr('dir', '__code__'):
-            # only on PyPy, CPython does not expose these attrs
-            raises(TypeError, "dir.__code__ = f.__code__")
-            raises(TypeError, "list().append.__func__.__code__ = f.__code__")
+        raises(TypeError, "dir.__code__ = f.__code__")
+        raises(TypeError, "list.append.__code__ = f.__code__")
+
+    def test_write_extra_attributes_builtin_forbidden(self):
+        raises(AttributeError, "dir.abcd = 5")
+        raises(AttributeError, "list.append.im_func.efgh = 6")
+        raises(AttributeError, "dir.__dict__")
+        raises(AttributeError, "dir.__dict__ = {}")
 
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -281,6 +281,8 @@
 
 def test_elidable_kinds():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
+    from rpython.rlib.objectmodel import compute_hash
+    from rpython.rlib.rsiphash import enable_siphash24
 
     @jit.elidable
     def f1(n, m):
@@ -293,12 +295,17 @@
         if n > m:
             raise ValueError
         return n + m
+    @jit.elidable
+    def f4(n, m):
+        return compute_hash(str(n) + str(m))
 
     def f(n, m):
         a = f1(n, m)
         b = f2(n, m)
         c = f3(n, m)
-        return a + len(b) + c
+        d = f4(n, m)
+        enable_siphash24()
+        return a + len(b) + c + d
 
     rtyper = support.annotate(f, [7, 9])
     jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
@@ -309,7 +316,8 @@
     for index, expected in [
             (0, EffectInfo.EF_ELIDABLE_CANNOT_RAISE),
             (1, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR),
-            (2, EffectInfo.EF_ELIDABLE_CAN_RAISE)]:
+            (2, EffectInfo.EF_ELIDABLE_CAN_RAISE),
+            (3, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR)]:
         call_op = f_graph.startblock.operations[index]
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)

From pypy.commits at gmail.com  Wed Feb  1 09:06:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 06:06:33 -0800 (PST)
Subject: [pypy-commit] pypy default: Forbid reading __dict__ or setting
 random attributes on functions that
Message-ID: <5891eb69.92d41c0a.1e98.3969@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89882:0d92840e04ff
Date: 2017-02-01 15:05 +0100
http://bitbucket.org/pypy/pypy/changeset/0d92840e04ff/

Log:	Forbid reading __dict__ or setting random attributes on functions
	that are 'not can_change_code', corresponding to CPython's built-in
	functions

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -198,10 +198,16 @@
 
     def getdict(self, space):
         if self.w_func_dict is None:
+            if not self.can_change_code:
+                raise oefmt(space.w_AttributeError,
+                            "cannot set extra attributes on built-in functions")
             self.w_func_dict = space.newdict(instance=True)
         return self.w_func_dict
 
     def setdict(self, space, w_dict):
+        if not self.can_change_code:
+            raise oefmt(space.w_AttributeError,
+                        "cannot set __dict__ on built-in functions")
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError,
                         "setting function's dictionary to a non-dict")
@@ -660,7 +666,7 @@
         Function.__init__(self, func.space, func.code, func.w_func_globals,
                           func.defs_w, func.closure, func.name)
         self.w_doc = func.w_doc
-        self.w_func_dict = func.w_func_dict
+        #self.w_func_dict = func.w_func_dict---nowadays, always None
         self.w_module = func.w_module
 
     def descr_builtinfunction__new__(space, w_subtype):
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -98,6 +98,12 @@
         raises(TypeError, "dir.func_code = f.func_code")
         raises(TypeError, "list.append.im_func.func_code = f.func_code")
 
+    def test_write_extra_attributes_builtin_forbidden(self):
+        raises(AttributeError, "dir.abcd = 5")
+        raises(AttributeError, "list.append.im_func.efgh = 6")
+        raises(AttributeError, "dir.__dict__")
+        raises(AttributeError, "dir.__dict__ = {}")
+
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")
         exec '''if 1:

From pypy.commits at gmail.com  Wed Feb  1 09:24:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 06:24:52 -0800 (PST)
Subject: [pypy-commit] pypy default: don't crash on c_int.from_buffer()
Message-ID: <5891efb4.09bb1c0a.66c7a.3b08@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89884:b0a257824f6a
Date: 2017-02-01 15:24 +0100
http://bitbucket.org/pypy/pypy/changeset/b0a257824f6a/

Log:	don't crash on c_int.from_buffer()

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -102,7 +102,11 @@
                 % (len(buf) + offset, size + offset))
         raw_addr = buf._pypy_raw_address()
         result = self.from_address(raw_addr)
-        result._ensure_objects()['ffffffff'] = obj
+        objects = result._ensure_objects()
+        if objects is not None:
+            objects['ffffffff'] = obj
+        else:   # case e.g. of a primitive type like c_int
+            result._objects = obj
         return result
 
     def from_buffer_copy(self, obj, offset=0):
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
@@ -24,6 +24,16 @@
         assert b[0] == "a"
         assert b[:] == "abc\0"
 
+    def test_from_buffer(self):
+        b1 = bytearray("abcde")
+        b = (c_char * 5).from_buffer(b1)
+        assert b[2] == "c"
+        #
+        b1 = bytearray("abcd")
+        b = c_int.from_buffer(b1)
+        assert b.value in (1684234849,   # little endian
+                           1633837924)   # big endian
+
     try:
         c_wchar
     except NameError:

From pypy.commits at gmail.com  Wed Feb  1 09:25:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 06:25:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5891efc7.c69bdf0a.b3cdd.d4af@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89885:761e1ebb572b
Date: 2017-02-01 15:24 +0100
http://bitbucket.org/pypy/pypy/changeset/761e1ebb572b/

Log:	hg merge default

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -92,7 +92,11 @@
                 % (buf.nbytes, offset + size))
         raw_addr = buf._pypy_raw_address() + offset
         result = self.from_address(raw_addr)
-        result._ensure_objects()['ffffffff'] = obj
+        objects = result._ensure_objects()
+        if objects is not None:
+            objects['ffffffff'] = obj
+        else:   # case e.g. of a primitive type like c_int
+            result._objects = obj
         return result
 
     def from_buffer_copy(self, obj, offset=0):
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
@@ -24,6 +24,16 @@
         assert b[0] == "a"
         assert b[:] == "abc\0"
 
+    def test_from_buffer(self):
+        b1 = bytearray("abcde")
+        b = (c_char * 5).from_buffer(b1)
+        assert b[2] == "c"
+        #
+        b1 = bytearray("abcd")
+        b = c_int.from_buffer(b1)
+        assert b.value in (1684234849,   # little endian
+                           1633837924)   # big endian
+
     try:
         c_wchar
     except NameError:

From pypy.commits at gmail.com  Wed Feb  1 09:58:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 06:58:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test_importlib no longer segfaults
 (instead, it shows an infinite list
Message-ID: <5891f7a6.4a86df0a.172f0.d8c5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89886:63225fef53d9
Date: 2017-02-01 15:58 +0100
http://bitbucket.org/pypy/pypy/changeset/63225fef53d9/

Log:	test_importlib no longer segfaults (instead, it shows an infinite
	list of failures)

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -261,7 +261,7 @@
     RegrTest('test_imaplib.py'),
     RegrTest('test_imghdr.py'),
     RegrTest('test_import'),
-    RegrTest('test_importlib', skip='XXX segfaults'),
+    RegrTest('test_importlib'),
     RegrTest('test_imp.py', core=True, usemodules='thread'),
     RegrTest('test_index.py'),
     RegrTest('test_inspect.py', usemodules="struct unicodedata"),

From pypy.commits at gmail.com  Wed Feb  1 10:00:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 07:00:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Make tests at least progress a bit
Message-ID: <5891f820.8887df0a.945be.dc6b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89887:2e887b7ebe14
Date: 2017-02-01 16:00 +0100
http://bitbucket.org/pypy/pypy/changeset/2e887b7ebe14/

Log:	Make tests at least progress a bit

diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -238,6 +238,12 @@
 # different than CPython's, always < 3000.  We get the 4-bytes string
 # here via a hack: MAGIC_NUMBER is set in the module from
 # module/_frozen_importlib/__init__.py before the module is executed.
+# FOR TESTS ONLY, we make it default to imp.get_magic().
+try:
+    MAGIC_NUMBER
+except NameError:
+    import imp
+    MAGIC_NUMBER = imp.get_magic()
 
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 

From pypy.commits at gmail.com  Wed Feb  1 10:10:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 07:10:18 -0800 (PST)
Subject: [pypy-commit] pypy default: Hrmph, hack more,
 otherwise some regular use cases raise AttributeError
Message-ID: <5891fa5a.d30f1c0a.5df49.5519@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89888:347b4430addd
Date: 2017-02-01 16:09 +0100
http://bitbucket.org/pypy/pypy/changeset/347b4430addd/

Log:	Hrmph, hack more, otherwise some regular use cases raise
	AttributeError

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -204,6 +204,11 @@
             self.w_func_dict = space.newdict(instance=True)
         return self.w_func_dict
 
+    def getdictvalue(self, space, attr):
+        if not self.can_change_code:
+            return None
+        return W_Root.getdictvalue(self, space, attr)
+
     def setdict(self, space, w_dict):
         if not self.can_change_code:
             raise oefmt(space.w_AttributeError,
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -103,6 +103,8 @@
         raises(AttributeError, "list.append.im_func.efgh = 6")
         raises(AttributeError, "dir.__dict__")
         raises(AttributeError, "dir.__dict__ = {}")
+        c = all.__call__   # this should work
+        assert c([4, 5, 6]) is True
 
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")

From pypy.commits at gmail.com  Wed Feb  1 10:10:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 07:10:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5891fa82.1282df0a.bb340.f4e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89889:82e074d421eb
Date: 2017-02-01 16:10 +0100
http://bitbucket.org/pypy/pypy/changeset/82e074d421eb/

Log:	hg merge default

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -213,6 +213,11 @@
             self.w_func_dict = space.newdict(instance=True)
         return self.w_func_dict
 
+    def getdictvalue(self, space, attr):
+        if not self.can_change_code:
+            return None
+        return W_Root.getdictvalue(self, space, attr)
+
     def setdict(self, space, w_dict):
         if not self.can_change_code:
             raise oefmt(space.w_AttributeError,
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -168,6 +168,8 @@
         raises(AttributeError, "list.append.im_func.efgh = 6")
         raises(AttributeError, "dir.__dict__")
         raises(AttributeError, "dir.__dict__ = {}")
+        c = all.__call__   # this should work
+        assert c([4, 5, 6]) is True
 
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")

From pypy.commits at gmail.com  Wed Feb  1 10:54:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 07:54:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Skip test (together with f38fce96ddce,
 next merge)
Message-ID: <5892049f.0e1d1c0a.60479.6ff1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89891:6e40d32a99a1
Date: 2017-02-01 16:53 +0100
http://bitbucket.org/pypy/pypy/changeset/6e40d32a99a1/

Log:	Skip test (together with f38fce96ddce, next merge)

diff --git a/lib-python/3/test/test_signal.py b/lib-python/3/test/test_signal.py
--- a/lib-python/3/test/test_signal.py
+++ b/lib-python/3/test/test_signal.py
@@ -358,6 +358,8 @@
 
         assert_python_ok('-c', code)
 
+    @support.impl_detail("pypy writes the message to fd 2, not to sys.stderr",
+                         pypy=False)
     @unittest.skipIf(_testcapi is None, 'need _testcapi')
     def test_wakeup_write_error(self):
         # Issue #16105: write() errors in the C signal handler should not

From pypy.commits at gmail.com  Wed Feb  1 10:54:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 07:54:04 -0800 (PST)
Subject: [pypy-commit] pypy default: Like CPython 3,
 write a message to stderr if we can't write to wakeup_fd.
Message-ID: <5892049c.50a4df0a.2bbd1.0aea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89890:f38fce96ddce
Date: 2017-02-01 16:49 +0100
http://bitbucket.org/pypy/pypy/changeset/f38fce96ddce/

Log:	Like CPython 3, write a message to stderr if we can't write to
	wakeup_fd. Unlike CPython, the message is sent to fileno 2, not back
	through sys.stderr.

diff --git a/rpython/rlib/test/test_rsignal.py b/rpython/rlib/test/test_rsignal.py
--- a/rpython/rlib/test/test_rsignal.py
+++ b/rpython/rlib/test/test_rsignal.py
@@ -1,4 +1,4 @@
-import os, py
+import os, py, errno
 from rpython.translator.c.test.test_genc import compile
 from rpython.rlib import rsignal
 
@@ -37,3 +37,24 @@
 def test_compile():
     fn = compile(test_simple, [])
     fn()
+
+def test_compile_wakeup_fd():
+    def fn():
+        rd, wr = os.pipe()
+        rsignal.pypysig_set_wakeup_fd(wr, False)
+        for i in range(3):
+            rsignal.pypysig_setflag(rsignal.SIGUSR1)
+            os.kill(os.getpid(), rsignal.SIGUSR1)
+            check(rsignal.SIGUSR1)
+            check(-1)
+            check(-1)
+        x = os.read(rd, 10)
+        assert x == chr(rsignal.SIGUSR1) * 3
+        #
+        rsignal.pypysig_set_wakeup_fd(rd, False)   # can't write there
+        os.kill(os.getpid(), rsignal.SIGUSR1)
+
+    fn = compile(fn, [], return_stderr=True)
+    stderr = fn()
+    assert stderr.endswith('Exception ignored when trying to write to the '
+                           'signal wakeup fd: Errno %d\n' % errno.EBADF)
diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c
--- a/rpython/translator/c/src/signals.c
+++ b/rpython/translator/c/src/signals.c
@@ -2,6 +2,7 @@
 
 #include <limits.h>
 #include <stdlib.h>
+#include <errno.h>
 #ifdef _WIN32
 #include <process.h>
 #include <io.h>
@@ -82,27 +83,51 @@
       }
 }
 
+static void write_str(int fd, const char *p)
+{
+    int i = 0;
+    while (p[i] != '\x00')
+        i++;
+    (void)write(fd, p, i);
+}
+
 static void signal_setflag_handler(int signum)
 {
     pypysig_pushback(signum);
 
-    if (wakeup_fd != -1) 
-      {
+    /* Warning, this logic needs to be async-signal-safe */
+    if (wakeup_fd != -1) {
 #ifndef _WIN32
         ssize_t res;
 #else
         int res;
 #endif
-	if (wakeup_with_nul_byte)
-	{
-	    res = write(wakeup_fd, "\0", 1);
-	} else {
-	    unsigned char byte = (unsigned char)signum;
-	    res = write(wakeup_fd, &byte, 1);
-	}
-
-        /* the return value is ignored here */
-      }
+        int old_errno = errno;
+     retry:
+        if (wakeup_with_nul_byte) {
+            res = write(wakeup_fd, "\0", 1);
+        } else {
+            unsigned char byte = (unsigned char)signum;
+            res = write(wakeup_fd, &byte, 1);
+        }
+        if (res < 0) {
+            unsigned int e = (unsigned int)errno;
+            char c[27], *p;
+            if (e == EINTR)
+                goto retry;
+            write_str(2, "Exception ignored when trying to write to the "
+                         "signal wakeup fd: Errno ");
+            p = c + sizeof(c);
+            *--p = 0;
+            *--p = '\n';
+            do {
+                *--p = '0' + e % 10;
+                e /= 10;
+            } while (e != 0);
+            write_str(2, p);
+        }
+        errno = old_errno;
+    }
 }
 
 void pypysig_setflag(int signum)

From pypy.commits at gmail.com  Wed Feb  1 11:03:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 08:03:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Skip a CPython-only test
Message-ID: <589206c7.812d1c0a.b3328.69e4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89892:c372dcec31df
Date: 2017-02-01 17:02 +0100
http://bitbucket.org/pypy/pypy/changeset/c372dcec31df/

Log:	Skip a CPython-only test

diff --git a/lib-python/3/test/_test_multiprocessing.py b/lib-python/3/test/_test_multiprocessing.py
--- a/lib-python/3/test/_test_multiprocessing.py
+++ b/lib-python/3/test/_test_multiprocessing.py
@@ -2795,6 +2795,7 @@
             self.assertTrue((arena != narena and nstart == 0) or
                             (stop == nstart))
 
+    @test.support.cpython_only
     def test_free_from_gc(self):
         # Check that freeing of blocks by the garbage collector doesn't deadlock
         # (issue #12352).

From pypy.commits at gmail.com  Wed Feb  1 11:33:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 01 Feb 2017 08:33:53 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Copy a bunch of macros from the CPython
 header
Message-ID: <58920df1.49acdf0a.9bc36.0c3c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89893:92709629b937
Date: 2017-02-01 16:33 +0000
http://bitbucket.org/pypy/pypy/changeset/92709629b937/

Log:	Copy a bunch of macros from the CPython header

diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,6 +7,30 @@
 
 #include <cpyext_unicodeobject.h>
 
+/* Fast access macros */
+#ifndef Py_LIMITED_API
+
+#define PyUnicode_WSTR_LENGTH(op) \
+    (PyUnicode_IS_COMPACT_ASCII(op) ?                  \
+     ((PyASCIIObject*)op)->length :                    \
+     ((PyCompactUnicodeObject*)op)->wstr_length)
+
+/* Returns the deprecated Py_UNICODE representation's size in code units
+   (this includes surrogate pairs as 2 units).
+   If the Py_UNICODE representation is not available, it will be computed
+   on request.  Use PyUnicode_GET_LENGTH() for the length in code points. */
+
+#define PyUnicode_GET_SIZE(op)                       \
+    (assert(PyUnicode_Check(op)),                    \
+     (((PyASCIIObject *)(op))->wstr) ?               \
+      PyUnicode_WSTR_LENGTH(op) :                    \
+      ((void)PyUnicode_AsUnicode((PyObject *)(op)),  \
+       assert(((PyASCIIObject *)(op))->wstr),        \
+       PyUnicode_WSTR_LENGTH(op)))
+
+#define PyUnicode_GET_DATA_SIZE(op) \
+    (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE)
+
 /* Alias for PyUnicode_AsUnicode().  This will create a wchar_t/Py_UNICODE
    representation on demand.  Using this macro is very inefficient now,
    try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
@@ -21,12 +45,280 @@
     ((const char *)(PyUnicode_AS_UNICODE(op)))
 
 
-PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs);
-PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...);
+/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */
 
+/* Values for PyASCIIObject.state: */
+
+/* Interning state. */
+#define SSTATE_NOT_INTERNED 0
+#define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+
+/* Return true if the string contains only ASCII characters, or 0 if not. The
+   string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be
+   ready. */
+#define PyUnicode_IS_ASCII(op)                   \
+    (assert(PyUnicode_Check(op)),                \
+     assert(PyUnicode_IS_READY(op)),             \
+     ((PyASCIIObject*)op)->state.ascii)
+
+/* Return true if the string is compact or 0 if not.
+   No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT(op) \
+    (((PyASCIIObject*)(op))->state.compact)
+
+/* Return true if the string is a compact ASCII string (use PyASCIIObject
+   structure), or 0 if not.  No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT_ASCII(op)                 \
+    (((PyASCIIObject*)op)->state.ascii && PyUnicode_IS_COMPACT(op))
+
+enum PyUnicode_Kind {
+/* String contains only wstr byte characters.  This is only possible
+   when the string was created with a legacy API and _PyUnicode_Ready()
+   has not been called yet.  */
+    PyUnicode_WCHAR_KIND = 0,
+/* Return values of the PyUnicode_KIND() macro: */
+    PyUnicode_1BYTE_KIND = 1,
+    PyUnicode_2BYTE_KIND = 2,
+    PyUnicode_4BYTE_KIND = 4
+};
+
+/* Return pointers to the canonical representation cast to unsigned char,
+   Py_UCS2, or Py_UCS4 for direct character access.
+   No checks are performed, use PyUnicode_KIND() before to ensure
+   these will work correctly. */
+
+#define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op))
+#define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op))
+#define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op))
+
+/* Return one of the PyUnicode_*_KIND values defined above. */
+#define PyUnicode_KIND(op) \
+    (assert(PyUnicode_Check(op)), \
+     assert(PyUnicode_IS_READY(op)),            \
+     ((PyASCIIObject *)(op))->state.kind)
+
+/* Return a void pointer to the raw unicode buffer. */
+#define _PyUnicode_COMPACT_DATA(op)                     \
+    (PyUnicode_IS_ASCII(op) ?                   \
+     ((void*)((PyASCIIObject*)(op) + 1)) :              \
+     ((void*)((PyCompactUnicodeObject*)(op) + 1)))
+
+#define _PyUnicode_NONCOMPACT_DATA(op)                  \
+    (assert(((PyUnicodeObject*)(op))->data),        \
+     ((((PyUnicodeObject *)(op))->data)))
+
+#define PyUnicode_DATA(op) \
+    (assert(PyUnicode_Check(op)), \
+     PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) :   \
+     _PyUnicode_NONCOMPACT_DATA(op))
+
+/* In the access macros below, "kind" may be evaluated more than once.
+   All other macro parameters are evaluated exactly once, so it is safe
+   to put side effects into them (such as increasing the index). */
+
+/* Write into the canonical representation, this macro does not do any sanity
+   checks and is intended for usage in loops.  The caller should cache the
+   kind and data pointers obtained from other macro calls.
+   index is the index in the string (starts at 0) and value is the new
+   code point value which should be written to that location. */
+#define PyUnicode_WRITE(kind, data, index, value) \
+    do { \
+        switch ((kind)) { \
+        case PyUnicode_1BYTE_KIND: { \
+            ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \
+            break; \
+        } \
+        case PyUnicode_2BYTE_KIND: { \
+            ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \
+            break; \
+        } \
+        default: { \
+            assert((kind) == PyUnicode_4BYTE_KIND); \
+            ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \
+        } \
+        } \
+    } while (0)
+
+/* Read a code point from the string's canonical representation.  No checks
+   or ready calls are performed. */
+#define PyUnicode_READ(kind, data, index) \
+    ((Py_UCS4) \
+    ((kind) == PyUnicode_1BYTE_KIND ? \
+        ((const Py_UCS1 *)(data))[(index)] : \
+        ((kind) == PyUnicode_2BYTE_KIND ? \
+            ((const Py_UCS2 *)(data))[(index)] : \
+            ((const Py_UCS4 *)(data))[(index)] \
+        ) \
+    ))
+
+/* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it
+   calls PyUnicode_KIND() and might call it twice.  For single reads, use
+   PyUnicode_READ_CHAR, for multiple consecutive reads callers should
+   cache kind and use PyUnicode_READ instead. */
+#define PyUnicode_READ_CHAR(unicode, index) \
+    (assert(PyUnicode_Check(unicode)),          \
+     assert(PyUnicode_IS_READY(unicode)),       \
+     (Py_UCS4)                                  \
+        (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \
+            ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \
+            (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \
+                ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \
+                ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \
+            ) \
+        ))
+
+/* Returns the length of the unicode string. The caller has to make sure that
+   the string has it's canonical representation set before calling
+   this macro.  Call PyUnicode_(FAST_)Ready to ensure that. */
+#define PyUnicode_GET_LENGTH(op)                \
+    (assert(PyUnicode_Check(op)),               \
+     assert(PyUnicode_IS_READY(op)),            \
+     ((PyASCIIObject *)(op))->length)
+
+
+/* Fast check to determine whether an object is ready. Equivalent to
+   PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */
+
+#define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready)
+
+/* PyUnicode_READY() does less work than _PyUnicode_Ready() in the best
+   case.  If the canonical representation is not yet set, it will still call
+   _PyUnicode_Ready().
+   Returns 0 on success and -1 on errors. */
+#define PyUnicode_READY(op)                        \
+    (assert(PyUnicode_Check(op)),                       \
+     (PyUnicode_IS_READY(op) ?                          \
+      0 : _PyUnicode_Ready((PyObject *)(op))))
+
+/* Return a maximum character value which is suitable for creating another
+   string based on op.  This is always an approximation but more efficient
+   than iterating over the string. */
+#define PyUnicode_MAX_CHAR_VALUE(op) \
+    (assert(PyUnicode_IS_READY(op)),                                    \
+     (PyUnicode_IS_ASCII(op) ?                                          \
+      (0x7f) :                                                          \
+      (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ?                     \
+       (0xffU) :                                                        \
+       (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ?                    \
+        (0xffffU) :                                                     \
+        (0x10ffffU)))))
+
+#endif
+
+/* --- Constants ---------------------------------------------------------- */
+
+/* This Unicode character will be used as replacement character during
+   decoding if the errors argument is set to "replace". Note: the
+   Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
+   Unicode 3.0. */
+
+#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD)
+
+/* === Public API ========================================================= */
+
+PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(
+    const char *format,   /* ASCII-encoded string  */
+    va_list vargs
+    );
+PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(
+    const char *format,   /* ASCII-encoded string  */
+    ...
+    );
+
+/* Use only if you know it's a string */
+#define PyUnicode_CHECK_INTERNED(op) \
+    (((PyASCIIObject *)(op))->state.interned)
+
+/* --- wchar_t support for platforms which support it --------------------- */
+
+#ifdef HAVE_WCHAR_H
+
+/* Convert the Unicode object to a wide character string. The output string
+   always ends with a nul character. If size is not NULL, write the number of
+   wide characters (excluding the null character) into *size.
+
+   Returns a buffer allocated by PyMem_Malloc() (use PyMem_Free() to free it)
+   on success. On error, returns NULL, *size is undefined and raises a
+   MemoryError. */
+
+PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString(
+    PyObject *unicode,          /* Unicode object */
+    Py_ssize_t *size            /* number of characters of the result */
+    );
+
+#endif
+
+/* === Builtin Codecs =====================================================
+
+   Many of these APIs take two arguments encoding and errors. These
+   parameters encoding and errors have the same semantics as the ones
+   of the builtin str() API.
+
+   Setting encoding to NULL causes the default encoding (UTF-8) to be used.
+
+   Error handling is set by errors which may also be set to NULL
+   meaning to use the default handling defined for the codec. Default
+   error handling for all builtin codecs is "strict" (ValueErrors are
+   raised).
+
+   The codecs all use a similar interface. Only deviation from the
+   generic ones are documented.
+
+*/
+
+/* --- Manage the default encoding ---------------------------------------- */
+
+/* Returns a pointer to the default encoding (UTF-8) of the
+   Unicode object unicode and the size of the encoded representation
+   in bytes stored in *size.
+
+   In case of an error, no *size is set.
+
+   This function caches the UTF-8 encoded string in the unicodeobject
+   and subsequent calls will return the same string.  The memory is released
+   when the unicodeobject is deallocated.
+
+   _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to
+   support the previous internal function with the same behaviour.
+
+   *** This API is for interpreter INTERNAL USE ONLY and will likely
+   *** be removed or changed in the future.
+
+   *** If you need to access the Unicode object as UTF-8 bytes string,
+   *** please use PyUnicode_AsUTF8String() instead.
+*/
+
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(char *) PyUnicode_AsUTF8AndSize(
+    PyObject *unicode,
+    Py_ssize_t *size);
+#define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize
+#endif
+
+/* Returns a pointer to the default encoding (UTF-8) of the
+   Unicode object unicode.
+
+   Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation
+   in the unicodeobject.
+
+   _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to
+   support the previous internal function with the same behaviour.
+
+   Use of this API is DEPRECATED since no size information can be
+   extracted from the returned data.
+
+   *** This API is for interpreter INTERNAL USE ONLY and will likely
+   *** be removed or changed for Python 3.1.
+
+   *** If you need to access the Unicode object as UTF-8 bytes string,
+   *** please use PyUnicode_AsUTF8String() instead.
+
+*/
+
+#ifndef Py_LIMITED_API
 #define _PyUnicode_AsString PyUnicode_AsUTF8
-
-PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size);
+#endif
 
 Py_LOCAL_INLINE(size_t) Py_UNICODE_strlen(const Py_UNICODE *u)
 {
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -115,6 +115,9 @@
 def set_ascii(py_obj, value):
     get_state(py_obj).c_ascii = cts.cast('unsigned int', value)
 
+def set_ready(py_obj, value):
+    get_state(py_obj).c_ready = cts.cast('unsigned int', value)
+
 def get_wbuffer(py_obj):
     py_obj = cts.cast('PyASCIIObject*', py_obj)
     return py_obj.c_wstr
@@ -253,31 +256,6 @@
     """Get the maximum ordinal for a Unicode character."""
     return runicode.UNICHR(runicode.MAXUNICODE)
 
- at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyUnicode_GET_DATA_SIZE(space, w_obj):
-    """Return the size of the object's internal buffer in bytes.  o has to be a
-    PyUnicodeObject (not checked)."""
-    return rffi.sizeof(Py_UNICODE) * PyUnicode_GET_SIZE(space, w_obj)
-
- at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyUnicode_GET_SIZE(space, w_obj):
-    """Return the size of the object.  obj is a PyUnicodeObject (not
-    checked)."""
-    return space.len_w(w_obj)
-
- at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
-def PyUnicode_GET_LENGTH(space, w_obj):
-    """Return the length of the Unicode string, in code points.
-    o has to be a Unicode object in the "canonical" representation
-    (not checked)."""
-    assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
-    return space.len_w(w_obj)
-
- at cpython_api([PyObject], rffi.INT, error=CANNOT_FAIL)
-def PyUnicode_IS_READY(space, w_obj):
-    # PyPy is always ready.
-    return space.w_True
-
 @cts.decl("int _PyUnicode_Ready(PyObject *unicode)", error=-1)
 def _PyUnicode_Ready(space, w_obj):
     assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
@@ -304,6 +282,7 @@
             set_ascii(py_obj, 0)
             set_utf8(py_obj, 0)
             set_utf8_len(py_obj, 0)
+        set_ready(py_obj, 1)
     elif maxchar < 65536:
         # XXX: assumes that sizeof(wchar_t) == 4
         ucs2_str = unicode_encode_utf_16(
@@ -314,6 +293,7 @@
         set_kind(py_obj, _2BYTE_KIND)
         set_utf8(py_obj, 0)
         set_utf8_len(py_obj, 0)
+        set_ready(py_obj, 1)
     else:
         # XXX: assumes that sizeof(wchar_t) == 4
         ucs4_data = get_wbuffer(py_obj)
@@ -322,6 +302,7 @@
         set_kind(py_obj, _4BYTE_KIND)
         set_utf8(py_obj, 0)
         set_utf8_len(py_obj, 0)
+        set_ready(py_obj, 1)
 
 
 @cpython_api([PyObject], rffi.CWCHARP)

From pypy.commits at gmail.com  Wed Feb  1 13:56:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 01 Feb 2017 10:56:09 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: fixes
Message-ID: <58922f49.da6d1c0a.98692.c3f7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89894:3f73fe10d0f6
Date: 2017-02-01 18:55 +0000
http://bitbucket.org/pypy/pypy/changeset/3f73fe10d0f6/

Log:	fixes

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -41,7 +41,7 @@
                  int result = 0;
 
                  if(PyUnicode_GetLength(s) != 11) {
-                     result = -PyUnicode_GetSize(s);
+                     result = -PyUnicode_GetLength(s);
                  }
                  Py_DECREF(s);
                  return PyLong_FromLong(result);
@@ -213,10 +213,8 @@
 
 class TestUnicode(BaseApiTest):
     def test_unicodeobject(self, space):
-        assert PyUnicode_GET_SIZE(space, space.wrap(u'sp&#228;m')) == 4
         assert PyUnicode_GetSize(space, space.wrap(u'sp&#228;m')) == 4
         unichar = rffi.sizeof(Py_UNICODE)
-        assert PyUnicode_GET_DATA_SIZE(space, space.wrap(u'sp&#228;m')) == 4 * unichar
 
         encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, ))
         w_default_encoding = space.call_function(
@@ -629,7 +627,7 @@
 
         assert space.eq_w(w_y, space.wrap(u"abcd"))
 
-        size = PyUnicode_GET_SIZE(space, w_x)
+        size = PyUnicode_GetSize(space, w_x)
         Py_UNICODE_COPY(space, target_chunk, x_chunk, size)
         w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size))
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -54,15 +54,14 @@
     """
     typedescr = get_typedescr(space.w_unicode.layout.typedef)
     py_obj = typedescr.allocate(space, space.w_unicode)
-    py_uni = rffi.cast(PyUnicodeObject, py_obj)
 
     buflen = length + 1
-    set_wsize(py_uni, length)
-    set_wbuffer(py_uni,
+    set_wsize(py_obj, length)
+    set_wbuffer(py_obj,
         lltype.malloc(
             rffi.CWCHARP.TO, buflen, flavor='raw', zero=True,
             add_memory_pressure=True))
-    return py_uni
+    return py_obj
 
 def unicode_attach(space, py_obj, w_obj, w_userdata=None):
     "Fills a newly allocated PyUnicodeObject with a unicode string"
@@ -74,8 +73,7 @@
     Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
     be modified after this call.
     """
-    py_uni = rffi.cast(PyUnicodeObject, py_obj)
-    s = rffi.wcharpsize2unicode(get_wbuffer(py_uni), get_wsize(py_uni))
+    s = rffi.wcharpsize2unicode(get_wbuffer(py_obj), get_wsize(py_obj))
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
     w_obj.__init__(s)
@@ -140,7 +138,7 @@
 
 def set_utf8(py_obj, buf):
     py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
-    py_obj.c_utf8 = cts.cast('char *', buf)
+    py_obj.c_utf8 = buf
 
 def get_wsize(py_obj):
     py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
@@ -156,7 +154,7 @@
 
 def set_data(py_obj, p_data):
     py_obj = cts.cast('PyUnicodeObject*', py_obj)
-    py_obj.c_data = cts.cast('void *', p_data)
+    py_obj.c_data = p_data
 
 
 @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
@@ -272,38 +270,36 @@
     if maxchar < 256:
         ucs1_data = rffi.str2charp(unicode_encode_latin_1(
             w_obj._value, len(w_obj._value), errors='strict'))
-        set_data(py_obj, ucs1_data)
+        set_data(py_obj, cts.cast('void*', ucs1_data))
         set_kind(py_obj, _1BYTE_KIND)
         if maxchar < 128:
             set_ascii(py_obj, 1)
-            set_utf8(py_obj, get_data(py_obj))
+            set_utf8(py_obj, cts.cast('char*', get_data(py_obj)))
             set_utf8_len(py_obj, get_wsize(py_obj))
         else:
             set_ascii(py_obj, 0)
-            set_utf8(py_obj, 0)
+            set_utf8(py_obj, cts.cast('char *', 0))
             set_utf8_len(py_obj, 0)
-        set_ready(py_obj, 1)
     elif maxchar < 65536:
         # XXX: assumes that sizeof(wchar_t) == 4
         ucs2_str = unicode_encode_utf_16(
             w_obj._value, len(w_obj._value), errors='strict')
         ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
-        set_data(py_obj, ucs2_data)
+        set_data(py_obj, cts.cast('void*', ucs2_data))
         set_len(py_obj, get_wsize(py_obj))
         set_kind(py_obj, _2BYTE_KIND)
-        set_utf8(py_obj, 0)
+        set_utf8(py_obj, cts.cast('char *', 0))
         set_utf8_len(py_obj, 0)
-        set_ready(py_obj, 1)
     else:
         # XXX: assumes that sizeof(wchar_t) == 4
         ucs4_data = get_wbuffer(py_obj)
-        set_data(py_obj, ucs4_data)
+        set_data(py_obj, cts.cast('void*', ucs4_data))
         set_len(py_obj, get_wsize(py_obj))
         set_kind(py_obj, _4BYTE_KIND)
-        set_utf8(py_obj, 0)
+        set_utf8(py_obj, cts.cast('char *', 0))
         set_utf8_len(py_obj, 0)
-        set_ready(py_obj, 1)
-
+    set_ready(py_obj, 1)
+    return 0
 
 @cpython_api([PyObject], rffi.CWCHARP)
 def PyUnicode_AsUnicode(space, ref):
@@ -322,15 +318,14 @@
 
 @cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)")
 def PyUnicode_AsUTF8(space, ref):
-    ref_unicode = rffi.cast(PyUnicodeObject, ref)
-    if not get_utf8(ref_unicode):
+    if not get_utf8(ref):
         # Copy unicode buffer
         w_unicode = from_ref(space, ref)
         w_encoded = unicodeobject.encode_object(space, w_unicode, "utf-8",
                                                 "strict")
         s = space.bytes_w(w_encoded)
-        set_utf8(ref_unicode, rffi.str2charp(s))
-    return get_utf8(ref_unicode)
+        set_utf8(ref, rffi.str2charp(s))
+    return get_utf8(ref)
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
 def PyUnicode_GetSize(space, ref):
@@ -346,13 +341,13 @@
         return space.len_w(w_obj)
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
-def PyUnicode_GetLength(space, w_unicode):
+def PyUnicode_GetLength(space, unicode):
     """Return the length of the Unicode object, in code points."""
     # XXX: this is a stub
-    if not PyUnicode_Check(space, w_unicode):
+    if not PyUnicode_Check(space, unicode):
         PyErr_BadArgument(space)
     #PyUnicode_READY(w_unicode)
-    return PyUnicode_GET_LENGTH(space, w_unicode)
+    return get_len(unicode)
 
 @cpython_api([PyObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_AsWideChar(space, ref, buf, size):
@@ -443,7 +438,7 @@
         s = rffi.wcharpsize2unicode(wchar_p, length)
         return make_ref(space, space.wrap(s))
     else:
-        return rffi.cast(PyObject, new_empty_unicode(space, length))
+        return new_empty_unicode(space, length)
 
 @cpython_api([CONST_WSTRING, Py_ssize_t], PyObject, result_is_ll=True)
 def PyUnicode_FromWideChar(space, wchar_p, length):
@@ -634,7 +629,7 @@
         return make_ref(space, PyUnicode_DecodeUTF8(
             space, s, size, lltype.nullptr(rffi.CCHARP.TO)))
     else:
-        return rffi.cast(PyObject, new_empty_unicode(space, size))
+        return new_empty_unicode(space, size)
 
 @cpython_api([rffi.INT_real], PyObject)
 def PyUnicode_FromOrdinal(space, ordinal):
@@ -649,8 +644,8 @@
 @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1)
 def PyUnicode_Resize(space, ref, newsize):
     # XXX always create a new string so far
-    py_uni = rffi.cast(PyUnicodeObject, ref[0])
-    if not get_wbuffer(py_uni):
+    py_obj = ref[0]
+    if not get_wbuffer(py_obj):
         raise oefmt(space.w_SystemError,
                     "PyUnicode_Resize called on already created string")
     try:
@@ -660,11 +655,11 @@
         ref[0] = lltype.nullptr(PyObject.TO)
         raise
     to_cp = newsize
-    oldsize = get_wsize(py_uni)
+    oldsize = get_wsize(py_obj)
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        get_wbuffer(py_newuni)[i] = get_wbuffer(py_uni)[i]
+        get_wbuffer(py_newuni)[i] = get_wbuffer(py_obj)[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newuni)
     return 0

From pypy.commits at gmail.com  Wed Feb  1 14:11:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 01 Feb 2017 11:11:26 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: fix translation
Message-ID: <589232de.9d711c0a.71d4d.c648@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89895:6cc49984574c
Date: 2017-02-01 19:10 +0000
http://bitbucket.org/pypy/pypy/changeset/6cc49984574c/

Log:	fix translation

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.runicode import unicode_encode_latin_1, unicode_encode_utf_16
+from rpython.rlib.rarithmetic import widen
 
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
@@ -258,7 +259,7 @@
 def _PyUnicode_Ready(space, w_obj):
     assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
     py_obj = as_pyobj(space, w_obj)
-    assert get_kind(py_obj) == WCHAR_KIND
+    assert widen(get_kind(py_obj)) == WCHAR_KIND
     maxchar = 0
     for c in w_obj._value:
         if ord(c) > maxchar:

From pypy.commits at gmail.com  Wed Feb  1 15:56:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 01 Feb 2017 12:56:52 -0800 (PST)
Subject: [pypy-commit] pypy default: Change the time at which we modify a
 graph with "hint_close_stack".
Message-ID: <58924b94.44641c0a.de544.ea18@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89896:4a7ae8e412f5
Date: 2017-02-01 21:56 +0100
http://bitbucket.org/pypy/pypy/changeset/4a7ae8e412f5/

Log:	Change the time at which we modify a graph with "hint_close_stack".
	Previously, it was at the time where we find a direct_call to it.
	Now it is when we're about to gctransform the graph itself.

	If we first find a function pointer to this function, then this
	change is important: without it, funcgen.py will hold the unmodified
	graph and enumerate dependencies from there, instead of the real
	dependencies of the modified graph. As pycket found, there are cases
	where it occurs.

diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -26,7 +26,6 @@
 
 class AsmGcRootFrameworkGCTransformer(BaseFrameworkGCTransformer):
     _asmgcc_save_restore_arguments = None
-    _seen_gctransformer_hint_close_stack = None
 
     def push_roots(self, hop, keep_current_args=False):
         livevars = self.get_livevars_for_roots(hop, keep_current_args)
@@ -50,29 +49,28 @@
         hop.genop("direct_call", [c_asm_nocollect, name])
 
     def gct_direct_call(self, hop):
+        # just a sanity check: if we find a fnptr with the hint on the
+        # _callable, then we'd also find the hint by looking only at the
+        # graph.  We'll actually change this graph only later, in
+        # start_transforming_graph().
         fnptr = hop.spaceop.args[0].value
         try:
             close_stack = fnptr._obj._callable._gctransformer_hint_close_stack_
         except AttributeError:
+            pass
+        else:
+            assert fnptr._obj.graph.func is fnptr._obj._callable
+        BaseFrameworkGCTransformer.gct_direct_call(self, hop)
+
+    def start_transforming_graph(self, graph):
+        try:
+            close_stack = graph.func._gctransformer_hint_close_stack_
+        except AttributeError:
             close_stack = False
         if close_stack:
-            self.handle_call_with_close_stack(hop)
-        else:
-            BaseFrameworkGCTransformer.gct_direct_call(self, hop)
+            self._transform_hint_close_stack(graph)
 
-    def handle_call_with_close_stack(self, hop):
-        fnptr = hop.spaceop.args[0].value
-        if self._seen_gctransformer_hint_close_stack is None:
-            self._seen_gctransformer_hint_close_stack = {}
-        if fnptr._obj.graph not in self._seen_gctransformer_hint_close_stack:
-            self._transform_hint_close_stack(fnptr)
-            self._seen_gctransformer_hint_close_stack[fnptr._obj.graph] = True
-        #
-        livevars = self.push_roots(hop)
-        self.default(hop)
-        self.pop_roots(hop, livevars)
-
-    def _transform_hint_close_stack(self, fnptr):
+    def _transform_hint_close_stack(self, graph):
         # We cannot easily pass variable amount of arguments of the call
         # across the call to the pypy_asm_stackwalk helper.  So we store
         # them away and restore them.  More precisely, we need to
@@ -83,8 +81,8 @@
         sradict = self._asmgcc_save_restore_arguments
         sra = []     # list of pointers to raw-malloced containers for args
         seen = {}
-        FUNC1 = lltype.typeOf(fnptr).TO
-        for TYPE in FUNC1.ARGS:
+        ARGS = [v.concretetype for v in graph.getargs()]
+        for TYPE in ARGS:
             if isinstance(TYPE, lltype.Ptr):
                 TYPE = llmemory.Address
             num = seen.get(TYPE, 0)
@@ -98,8 +96,10 @@
             sra.append(sradict[key])
         #
         # make a copy of the graph that will reload the values
-        graph = fnptr._obj.graph
         graph2 = copygraph(graph)
+        del graph2.func   # otherwise, start_transforming_graph() will
+                          # again transform graph2, and we get an
+                          # infinite loop
         #
         # edit the original graph to only store the value of the arguments
         block = Block(graph.startblock.inputargs)
@@ -116,17 +116,18 @@
                 SpaceOperation("bare_setfield", [c_p, c_item0, v_arg], v_void))
         #
         # call asm_stackwalk(graph2)
-        FUNC2 = lltype.FuncType([], FUNC1.RESULT)
+        RESULT = graph.getreturnvar().concretetype
+        FUNC2 = lltype.FuncType([], RESULT)
         fnptr2 = lltype.functionptr(FUNC2,
-                                    fnptr._obj._name + '_reload',
+                                    graph.name + '_reload',
                                     graph=graph2)
         c_fnptr2 = Constant(fnptr2, lltype.Ptr(FUNC2))
         HELPERFUNC = lltype.FuncType([lltype.Ptr(FUNC2),
-                                      ASM_FRAMEDATA_HEAD_PTR], FUNC1.RESULT)
+                                      ASM_FRAMEDATA_HEAD_PTR], RESULT)
         v_asm_stackwalk = varoftype(lltype.Ptr(HELPERFUNC), "asm_stackwalk")
         block.operations.append(
             SpaceOperation("cast_pointer", [c_asm_stackwalk], v_asm_stackwalk))
-        v_result = varoftype(FUNC1.RESULT)
+        v_result = varoftype(RESULT)
         block.operations.append(
             SpaceOperation("indirect_call", [v_asm_stackwalk, c_fnptr2,
                                              c_gcrootanchor,
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -201,6 +201,9 @@
         self.var_last_needed_in = None
         self.curr_block = None
 
+    def start_transforming_graph(self, graph):
+        pass    # for asmgcc.py
+
     def transform_graph(self, graph):
         if graph in self.minimal_transform:
             if self.minimalgctransformer:
@@ -210,6 +213,7 @@
         if graph in self.seen_graphs:
             return
         self.seen_graphs.add(graph)
+        self.start_transforming_graph(graph)
 
         self.links_to_split = {} # link -> vars to pop_alive across the link
 

From pypy.commits at gmail.com  Thu Feb  2 04:12:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 02 Feb 2017 01:12:18 -0800 (PST)
Subject: [pypy-commit] pypy default: Move some methods from the W_Array
 subclasses (there are more than 10 of them) to the base class. This reduces
 by 40% the size of pypy_module_array*.c.
Message-ID: <5892f7f2.eb88df0a.0977.506c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89897:a9ea4365a378
Date: 2017-02-02 10:11 +0100
http://bitbucket.org/pypy/pypy/changeset/a9ea4365a378/

Log:	Move some methods from the W_Array subclasses (there are more than
	10 of them) to the base class. This reduces by 40% the size of
	pypy_module_array*.c.

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -122,12 +122,120 @@
                                         hints={'nolength': True}))
 
 class W_ArrayBase(W_Root):
-    _attrs_ = ('space', 'len', 'allocated', '_lifeline_') # no buffer
+    _attrs_ = ('space', 'len', 'allocated', '_lifeline_', '_buffer')
 
     def __init__(self, space):
         self.space = space
         self.len = 0
         self.allocated = 0
+        self._buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+
+    def setlen(self, size, zero=False, overallocate=True):
+        if size > 0:
+            if size > self.allocated or size < self.allocated / 2:
+                if overallocate:
+                    if size < 9:
+                        some = 3
+                    else:
+                        some = 6
+                    some += size >> 3
+                else:
+                    some = 0
+                self.allocated = size + some
+                byte_size = self.allocated * self.itemsize
+                if zero:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True, zero=True)
+                else:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True)
+                    copy_bytes = min(size, self.len) * self.itemsize
+                    rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer),
+                                  rffi.cast(rffi.VOIDP, self._buffer),
+                                  copy_bytes)
+            else:
+                self.len = size
+                return
+        else:
+            assert size == 0
+            self.allocated = 0
+            new_buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+        self._buffer = new_buffer
+        self.len = size
+
+    def _fromiterable(self, w_seq):
+        # used by fromsequence().
+        # a more careful case if w_seq happens to be a very large
+        # iterable: don't copy the items into some intermediate list
+        w_iterator = self.space.iter(w_seq)
+        tp = self.space.type(w_iterator)
+        while True:
+            unpack_driver.jit_merge_point(selfclass=self.__class__,
+                                          tp=tp, self=self,
+                                          w_iterator=w_iterator)
+            space = self.space
+            try:
+                w_item = space.next(w_iterator)
+            except OperationError as e:
+                if not e.match(space, space.w_StopIteration):
+                    raise
+                break  # done
+            self.descr_append(space, w_item)
+
+    def _charbuf_start(self):
+        return self._buffer
+
+    def _buffer_as_unsigned(self):
+        return rffi.cast(lltype.Unsigned, self._buffer)
+
+    def _charbuf_stop(self):
+        keepalive_until_here(self)
+
+    def delitem(self, space, i, j):
+        if i < 0:
+            i += self.len
+        if i < 0:
+            i = 0
+        if j < 0:
+            j += self.len
+        if j < 0:
+            j = 0
+        if j > self.len:
+            j = self.len
+        if i >= j:
+            return None
+        oldbuffer = self._buffer
+        self._buffer = lltype.malloc(rffi.CCHARP.TO,
+            (self.len - (j - i)) * self.itemsize, flavor='raw',
+            add_memory_pressure=True)
+        if i:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, self._buffer),
+                rffi.cast(rffi.VOIDP, oldbuffer),
+                i * self.itemsize
+            )
+        if j < self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                                  i * self.itemsize)),
+                rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer,
+                                                  j * self.itemsize)),
+                (self.len - j) * self.itemsize
+            )
+        self.len -= j - i
+        self.allocated = self.len
+        if oldbuffer:
+            lltype.free(oldbuffer, flavor='raw')
 
     def readbuf_w(self, space):
         return ArrayBuffer(self, True)
@@ -154,14 +262,24 @@
 
         Return number of occurrences of x in the array.
         """
-        raise NotImplementedError
+        cnt = 0
+        for i in range(self.len):
+            # XXX jitdriver
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_val)):
+                cnt += 1
+        return space.wrap(cnt)
 
     def descr_index(self, space, w_x):
         """ index(x)
 
         Return index of first occurrence of x in the array.
         """
-        raise NotImplementedError
+        for i in range(self.len):
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_x)):
+                return space.wrap(i)
+        raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
     def descr_reverse(self, space):
         """ reverse()
@@ -175,7 +293,8 @@
 
         Remove the first occurrence of x in the array.
         """
-        raise NotImplementedError
+        w_idx = self.descr_index(space, w_val)
+        self.descr_pop(space, space.int_w(w_idx))
 
     @unwrap_spec(i=int)
     def descr_pop(self, space, i=-1):
@@ -453,16 +572,102 @@
         return space.newseqiter(self)
 
     def descr_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        a = self.constructor(space)
+        a.setlen(self.len + w_other.len, overallocate=False)
+        if self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, a._buffer),
+                rffi.cast(rffi.VOIDP, self._buffer),
+                self.len * self.itemsize
+            )
+        if w_other.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(a._buffer,
+                                             self.len * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                w_other.len * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_inplace_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        oldlen = self.len
+        otherlen = w_other.len
+        self.setlen(oldlen + otherlen)
+        if otherlen:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                             oldlen * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                otherlen * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(w_other)
+        return self
+
+    def _mul_helper(self, space, w_repeat, is_inplace):
+        try:
+            repeat = space.getindex_w(w_repeat, space.w_OverflowError)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
+        if is_inplace:
+            a = self
+            start = 1
+        else:
+            a = self.constructor(space)
+            start = 0
+        if repeat <= start:
+            if repeat <= 0:
+                a.setlen(0, overallocate=False)
+            return a
+        oldlen = self.len
+        try:
+            newlen = ovfcheck(oldlen * repeat)
+        except OverflowError:
+            raise MemoryError
+        #
+        srcbuf = self._buffer
+        srcsize = self.len * self.itemsize
+        for i in range(srcsize):
+            if srcbuf[i] != '\x00':
+                break
+        else:
+            # the source is entirely zero: initialize the target
+            # with zeroes too
+            a.setlen(newlen, zero=True, overallocate=False)
+            return a
+        #
+        a.setlen(newlen, overallocate=False)
+        srcbuf = self._buffer   # reload this, in case self is a
+        if oldlen == 1:
+            self._repeat_single_item(a, start, repeat)
+        else:
+            dstbuf = a._buffer
+            if start == 1:
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+            for r in range(start, repeat):
+                rffi.c_memcpy(rffi.cast(rffi.VOIDP, dstbuf),
+                              rffi.cast(rffi.VOIDP, srcbuf),
+                              srcsize)
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, False)
 
     def descr_inplace_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, True)
 
     def descr_radd(self, space, w_other):
         return self.descr_add(space, w_other)
@@ -551,6 +756,7 @@
         self.itemtype = itemtype
         self.bytes = rffi.sizeof(itemtype)
         self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
+        self.arrayptrtype = lltype.Ptr(self.arraytype)
         self.unwrap = unwrap
         self.signed = signed
         self.canoverflow = canoverflow
@@ -640,22 +846,21 @@
         return self.array._charbuf_start()
 
 
+unpack_driver = jit.JitDriver(name='unpack_array',
+                              greens=['selfclass', 'tp'],
+                              reds=['self', 'w_iterator'])
+
 def make_array(mytype):
     W_ArrayBase = globals()['W_ArrayBase']
 
-    unpack_driver = jit.JitDriver(name='unpack_array',
-                                  greens=['tp'],
-                                  reds=['self', 'w_iterator'])
-
     class W_Array(W_ArrayBase):
         itemsize = mytype.bytes
         typecode = mytype.typecode
 
-        _attrs_ = ('space', 'len', 'allocated', '_lifeline_', 'buffer')
+        _attrs_ = W_ArrayBase._attrs_
 
-        def __init__(self, space):
-            W_ArrayBase.__init__(self, space)
-            self.buffer = lltype.nullptr(mytype.arraytype)
+        def get_buffer(self):
+            return rffi.cast(mytype.arrayptrtype, self._buffer)
 
         def item_w(self, w_item):
             space = self.space
@@ -709,46 +914,6 @@
                                          self.space.wrap(msg))
             return result
 
-        @rgc.must_be_light_finalizer
-        def __del__(self):
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-
-        def setlen(self, size, zero=False, overallocate=True):
-            if size > 0:
-                if size > self.allocated or size < self.allocated / 2:
-                    if overallocate:
-                        if size < 9:
-                            some = 3
-                        else:
-                            some = 6
-                        some += size >> 3
-                    else:
-                        some = 0
-                    self.allocated = size + some
-                    if zero:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True, zero=True)
-                    else:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True)
-                        for i in range(min(size, self.len)):
-                            new_buffer[i] = self.buffer[i]
-                else:
-                    self.len = size
-                    return
-            else:
-                assert size == 0
-                self.allocated = 0
-                new_buffer = lltype.nullptr(mytype.arraytype)
-
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-            self.buffer = new_buffer
-            self.len = size
-
         def fromsequence(self, w_seq):
             space = self.space
             oldlen = self.len
@@ -764,20 +929,21 @@
             if lst is not None:
                 self.setlen(oldlen + len(lst))
                 try:
-                    buf = self.buffer
+                    buf = self.get_buffer()
                     for num in lst:
                         buf[newlen] = self.item_from_int_or_float(num)
                         newlen += 1
                 except OperationError:
                     self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             # this is the common case: w_seq is a list or a tuple
             lst_w = space.listview_no_unpack(w_seq)
             if lst_w is not None:
                 self.setlen(oldlen + len(lst_w))
-                buf = self.buffer
+                buf = self.get_buffer()
                 try:
                     for w_num in lst_w:
                         # note: self.item_w() might invoke arbitrary code.
@@ -788,30 +954,14 @@
                         buf[newlen] = self.item_w(w_num)
                         newlen += 1
                 except OperationError:
-                    if buf == self.buffer:
+                    if buf == self.get_buffer():
                         self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             self._fromiterable(w_seq)
 
-        def _fromiterable(self, w_seq):
-            # a more careful case if w_seq happens to be a very large
-            # iterable: don't copy the items into some intermediate list
-            w_iterator = self.space.iter(w_seq)
-            tp = self.space.type(w_iterator)
-            while True:
-                unpack_driver.jit_merge_point(tp=tp, self=self,
-                                              w_iterator=w_iterator)
-                space = self.space
-                try:
-                    w_item = space.next(w_iterator)
-                except OperationError as e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    break  # done
-                self.descr_append(space, w_item)
-
         def extend(self, w_iterable, accept_different_array=False):
             space = self.space
             if isinstance(w_iterable, W_Array):
@@ -819,11 +969,14 @@
                 new = w_iterable.len
                 self.setlen(self.len + new)
                 i = 0
+                buf = self.get_buffer()
+                srcbuf = w_iterable.get_buffer()
                 while i < new:
                     if oldlen + i >= self.len:
                         self.setlen(oldlen + i + 1)
-                    self.buffer[oldlen + i] = w_iterable.buffer[i]
+                    buf[oldlen + i] = srcbuf[i]
                     i += 1
+                keepalive_until_here(w_iterable)
                 self.setlen(oldlen + i)
             elif (not accept_different_array
                   and isinstance(w_iterable, W_ArrayBase)):
@@ -832,17 +985,9 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def _charbuf_start(self):
-            return rffi.cast(rffi.CCHARP, self.buffer)
-
-        def _buffer_as_unsigned(self):
-            return rffi.cast(lltype.Unsigned, self.buffer)
-
-        def _charbuf_stop(self):
+        def w_getitem(self, space, idx):
+            item = self.get_buffer()[idx]
             keepalive_until_here(self)
-
-        def w_getitem(self, space, idx):
-            item = self.buffer[idx]
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
             elif mytype.typecode == 'f':
@@ -855,29 +1000,16 @@
             x = self.item_w(w_x)
             index = self.len
             self.setlen(index + 1)
-            self.buffer[index] = x
+            self.get_buffer()[index] = x
+            keepalive_until_here(self)
 
         # List interface
-        def descr_count(self, space, w_val):
-            cnt = 0
-            for i in range(self.len):
-                # XXX jitdriver
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    cnt += 1
-            return space.wrap(cnt)
-
-        def descr_index(self, space, w_val):
-            for i in range(self.len):
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    return space.wrap(i)
-            raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
         def descr_reverse(self, space):
-            b = self.buffer
+            b = self.get_buffer()
             for i in range(self.len / 2):
                 b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i]
+            keepalive_until_here(self)
 
         def descr_pop(self, space, i):
             if i < 0:
@@ -885,16 +1017,14 @@
             if i < 0 or i >= self.len:
                 raise oefmt(space.w_IndexError, "pop index out of range")
             w_val = self.w_getitem(space, i)
+            b = self.get_buffer()
             while i < self.len - 1:
-                self.buffer[i] = self.buffer[i + 1]
+                b[i] = b[i + 1]
                 i += 1
+            keepalive_until_here(self)
             self.setlen(self.len - 1)
             return w_val
 
-        def descr_remove(self, space, w_val):
-            w_idx = self.descr_index(space, w_val)
-            self.descr_pop(space, space.int_w(w_idx))
-
         def descr_insert(self, space, idx, w_val):
             if idx < 0:
                 idx += self.len
@@ -906,10 +1036,12 @@
             val = self.item_w(w_val)
             self.setlen(self.len + 1)
             i = self.len - 1
+            b = self.get_buffer()
             while i > idx:
-                self.buffer[i] = self.buffer[i - 1]
+                b[i] = b[i - 1]
                 i -= 1
-            self.buffer[i] = val
+            b[i] = val
+            keepalive_until_here(self)
 
         def getitem_slice(self, space, w_idx):
             start, stop, step, size = space.decode_index4(w_idx, self.len)
@@ -917,9 +1049,13 @@
             w_a.setlen(size, overallocate=False)
             assert step != 0
             j = 0
+            buf = w_a.get_buffer()
+            srcbuf = self.get_buffer()
             for i in range(start, stop, step):
-                w_a.buffer[j] = self.buffer[i]
+                buf[j] = srcbuf[i]
                 j += 1
+            keepalive_until_here(self)
+            keepalive_until_here(w_a)
             return w_a
 
         def setitem(self, space, w_idx, w_item):
@@ -928,7 +1064,8 @@
                 raise oefmt(self.space.w_TypeError,
                             "can only assign array to array slice")
             item = self.item_w(w_item)
-            self.buffer[idx] = item
+            self.get_buffer()[idx] = item
+            keepalive_until_here(self)
 
         def setitem_slice(self, space, w_idx, w_item):
             if not isinstance(w_item, W_Array):
@@ -945,127 +1082,21 @@
                 self.fromsequence(w_lst)
             else:
                 j = 0
+                buf = self.get_buffer()
+                srcbuf = w_item.get_buffer()
                 for i in range(start, stop, step):
-                    self.buffer[i] = w_item.buffer[j]
+                    buf[i] = srcbuf[j]
                     j += 1
+                keepalive_until_here(w_item)
+                keepalive_until_here(self)
 
-        def delitem(self, space, i, j):
-            if i < 0:
-                i += self.len
-            if i < 0:
-                i = 0
-            if j < 0:
-                j += self.len
-            if j < 0:
-                j = 0
-            if j > self.len:
-                j = self.len
-            if i >= j:
-                return None
-            oldbuffer = self.buffer
-            self.buffer = lltype.malloc(
-                mytype.arraytype, max(self.len - (j - i), 0), flavor='raw',
-                add_memory_pressure=True)
-            if i:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, self.buffer),
-                    rffi.cast(rffi.VOIDP, oldbuffer),
-                    i * mytype.bytes
-                )
-            if j < self.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, i)),
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer, j)),
-                    (self.len - j) * mytype.bytes
-                )
-            self.len -= j - i
-            self.allocated = self.len
-            if oldbuffer:
-                lltype.free(oldbuffer, flavor='raw')
-
-        # Add and mul methods
-        def descr_add(self, space, w_other):
-            if not isinstance(w_other, W_Array):
-                return space.w_NotImplemented
-            a = mytype.w_class(space)
-            a.setlen(self.len + w_other.len, overallocate=False)
-            if self.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, a.buffer),
-                    rffi.cast(rffi.VOIDP, self.buffer),
-                    self.len * mytype.bytes
-                )
-            if w_other.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(a.buffer, self.len)),
-                    rffi.cast(rffi.VOIDP, w_other.buffer),
-                    w_other.len * mytype.bytes
-                )
-            return a
-
-        def descr_inplace_add(self, space, w_other):
-            if not isinstance(w_other, W_Array):
-                return space.w_NotImplemented
-            oldlen = self.len
-            otherlen = w_other.len
-            self.setlen(oldlen + otherlen)
-            if otherlen:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, oldlen)),
-                    rffi.cast(rffi.VOIDP, w_other.buffer),
-                    otherlen * mytype.bytes
-                )
-            return self
-
-        def descr_mul(self, space, w_repeat):
-            return _mul_helper(space, self, w_repeat, False)
-
-        def descr_inplace_mul(self, space, w_repeat):
-            return _mul_helper(space, self, w_repeat, True)
-
-    def _mul_helper(space, self, w_repeat, is_inplace):
-        try:
-            repeat = space.getindex_w(w_repeat, space.w_OverflowError)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        repeat = max(repeat, 0)
-        try:
-            newlen = ovfcheck(self.len * repeat)
-        except OverflowError:
-            raise MemoryError
-        oldlen = self.len
-        if is_inplace:
-            a = self
-            start = 1
-        else:
-            a = mytype.w_class(space)
-            start = 0
-        # <a performance hack>
-        if oldlen == 1:
-            if mytype.unwrap == 'str_w' or mytype.unwrap == 'unicode_w':
-                zero = not ord(self.buffer[0])
-            elif mytype.unwrap == 'int_w' or mytype.unwrap == 'bigint_w':
-                zero = not widen(self.buffer[0])
-            #elif mytype.unwrap == 'float_w':
-            #    value = ...float(self.buffer[0])  xxx handle the case of -0.0
-            else:
-                zero = False
-            if zero:
-                a.setlen(newlen, zero=True, overallocate=False)
-                return a
-            a.setlen(newlen, overallocate=False)
-            item = self.buffer[0]
+        def _repeat_single_item(self, a, start, repeat):
+            # <a performance hack>
+            assert isinstance(a, W_Array)
+            item = self.get_buffer()[0]
+            dstbuf = a.get_buffer()
             for r in range(start, repeat):
-                a.buffer[r] = item
-            return a
-        # </a performance hack>
-        a.setlen(newlen, overallocate=False)
-        for r in range(start, repeat):
-            for i in range(oldlen):
-                a.buffer[r * oldlen + i] = self.buffer[i]
-        return a
+                dstbuf[r] = item
 
     mytype.w_class = W_Array
     W_Array.constructor = W_Array

From pypy.commits at gmail.com  Thu Feb  2 08:35:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 02 Feb 2017 05:35:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Hard to test: add
 rstack.stack_almost_full() and use it:
Message-ID: <5893359f.ddaadf0a.923d8.0994@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89898:c48c3b08c41f
Date: 2017-02-02 14:34 +0100
http://bitbucket.org/pypy/pypy/changeset/c48c3b08c41f/

Log:	Hard to test: add rstack.stack_almost_full() and use it:

	* to not start JITting when the stack is almost full

	* to propagate (instead of catching) a RuntimeError caused by stack
	overflow as long as the stack is still almost full

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,7 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rlib import rstackovf
+from rpython.rlib import rstack, rstackovf
 
 from pypy.interpreter import debug
 
@@ -55,6 +55,7 @@
         "Check if this is an exception that should better not be caught."
         return (self.match(space, space.w_SystemExit) or
                 self.match(space, space.w_KeyboardInterrupt))
+        # note: an extra case is added in OpErrFmtNoArgs
 
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
@@ -387,6 +388,16 @@
     def _compute_value(self, space):
         return self._value
 
+    def async(self, space):
+        # also matches a RuntimeError("maximum rec.") if the stack is
+        # still almost full, because in this case it might be a better
+        # idea to propagate the exception than eat it
+        if (self.w_type is space.w_RuntimeError and
+            self._value == "maximum recursion depth exceeded" and
+            rstack.stack_almost_full()):
+            return True
+        return OperationError.async(self, space)
+
 @specialize.memo()
 def get_operr_class(valuefmt):
     try:
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -88,6 +88,7 @@
                           'interp_magic.save_module_content_for_future_reload',
         'decode_long'               : 'interp_magic.decode_long',
         '_promote'                   : 'interp_magic._promote',
+        'stack_almost_full'         : 'interp_magic.stack_almost_full',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -9,7 +9,7 @@
 from pypy.objspace.std.setobject import W_BaseSetObject
 from pypy.objspace.std.typeobject import MethodCache
 from pypy.objspace.std.mapdict import MapAttrCache
-from rpython.rlib import rposix, rgc
+from rpython.rlib import rposix, rgc, rstack
 
 
 def internal_repr(space, w_object):
@@ -190,3 +190,7 @@
     else:
         jit.promote(w_obj)
     return w_obj
+
+def stack_almost_full(space):
+    """Return True if the stack is more than 15/16th full."""
+    return space.wrap(rstack.stack_almost_full())
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -732,7 +732,8 @@
     TY_FLOAT        = 0x06
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
-        if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
+        if (self.must_compile(deadframe, metainterp_sd, jitdriver_sd)
+                and not rstack.stack_almost_full()):
             self.start_compiling()
             try:
                 self._trace_and_compile_from_bridge(deadframe, metainterp_sd,
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -11,6 +11,7 @@
 from rpython.rlib.objectmodel import specialize, we_are_translated, r_dict
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib import rstack
 from rpython.rtyper.annlowlevel import (hlstr, cast_base_ptr_to_instance,
     cast_object_to_ptr)
 from rpython.rtyper.lltypesystem import lltype, llmemory, rstr, rffi
@@ -415,6 +416,8 @@
             if not confirm_enter_jit(*args):
                 return
             jitcounter.decay_all_counters()
+            if rstack.stack_almost_full():
+                return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -69,3 +69,22 @@
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
 stack_check_slowpath._dont_insert_stackcheck_ = True
+
+def stack_almost_full():
+    """Return True if the stack is more than 15/16th full."""
+    if not we_are_translated():
+        return False
+    # see stack_check()
+    current = llop.stack_current(lltype.Signed)
+    end = _stack_get_end()
+    length = 15 * (r_uint(_stack_get_length()) >> 4)
+    ofs = r_uint(end - current)
+    if ofs <= length:
+        return False    # fine
+    else:
+        _stack_too_big_slowpath(current)   # this might update the stack end
+        end = _stack_get_end()
+        ofs = r_uint(end - current)
+        return ofs > length
+stack_almost_full._dont_insert_stackcheck_ = True
+stack_almost_full._jit_look_inside_ = False

From pypy.commits at gmail.com  Thu Feb  2 10:02:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 02 Feb 2017 07:02:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default,
 with some fixes for the merge
Message-ID: <589349fd.2a99df0a.43838.e7ad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89899:066e6b9a9b2d
Date: 2017-02-02 15:59 +0100
http://bitbucket.org/pypy/pypy/changeset/066e6b9a9b2d/

Log:	hg merge default, with some fixes for the merge

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -225,14 +225,14 @@
     def buffer_w(self, space, flags):
         if _does_override_buffer_w_ex(self.__class__):
             return self.buffer_w_ex(space, flags)[0]
-        return self._buffer(space, flags).buffer_w(space, flags)
+        return self.__buffer_w(space, flags).buffer_w(space, flags)
 
     def buffer_w_ex(self, space, flags):
         if _does_override_buffer_w(self.__class__):
             return self.buffer_w(space, flags), 'B', 1
-        return self._buffer(space, flags).buffer_w_ex(space, flags)
+        return self.__buffer_w(space, flags).buffer_w_ex(space, flags)
 
-    def _buffer(self, space, flags):
+    def __buffer_w(self, space, flags):
         if flags & space.BUF_WRITABLE:
             w_impl = space.lookup(self, '__wbuffer__')
         else:
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,7 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rlib import rstackovf
+from rpython.rlib import rstack, rstackovf
 
 from pypy.interpreter import debug
 
@@ -60,6 +60,7 @@
         "Check if this is an exception that should better not be caught."
         return (self.match(space, space.w_SystemExit) or
                 self.match(space, space.w_KeyboardInterrupt))
+        # note: an extra case is added in OpErrFmtNoArgs
 
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
@@ -512,6 +513,16 @@
     def _compute_value(self, space):
         return self._value.decode('utf-8')
 
+    def async(self, space):
+        # also matches a RuntimeError("maximum rec.") if the stack is
+        # still almost full, because in this case it might be a better
+        # idea to propagate the exception than eat it
+        if (self.w_type is space.w_RecursionError and
+            self._value == "maximum recursion depth exceeded" and
+            rstack.stack_almost_full()):
+            return True
+        return OperationError.async(self, space)
+
 @specialize.memo()
 def get_operr_class(valuefmt):
     try:
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -88,6 +88,7 @@
         '_promote'                   : 'interp_magic._promote',
         'normalize_exc'             : 'interp_magic.normalize_exc',
         'StdErrPrinter'             : 'interp_stderrprinter.W_StdErrPrinter',
+        'stack_almost_full'         : 'interp_magic.stack_almost_full',
     }
 
     submodules = {
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -9,7 +9,7 @@
 from pypy.objspace.std.setobject import W_BaseSetObject
 from pypy.objspace.std.typeobject import MethodCache
 from pypy.objspace.std.mapdict import MapAttrCache
-from rpython.rlib import rposix, rgc
+from rpython.rlib import rposix, rgc, rstack
 
 
 def internal_repr(space, w_object):
@@ -176,3 +176,7 @@
     operr = OperationError(w_type, w_value, w_tb)
     operr.normalize_exception(space)
     return operr.get_w_value(space)
+
+def stack_almost_full(space):
+    """Return True if the stack is more than 15/16th full."""
+    return space.wrap(rstack.stack_almost_full())
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -141,12 +141,120 @@
                                         hints={'nolength': True}))
 
 class W_ArrayBase(W_Root):
-    _attrs_ = ('space', 'len', 'allocated', '_lifeline_') # no buffer
+    _attrs_ = ('space', 'len', 'allocated', '_lifeline_', '_buffer')
 
     def __init__(self, space):
         self.space = space
         self.len = 0
         self.allocated = 0
+        self._buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+
+    def setlen(self, size, zero=False, overallocate=True):
+        if size > 0:
+            if size > self.allocated or size < self.allocated / 2:
+                if overallocate:
+                    if size < 9:
+                        some = 3
+                    else:
+                        some = 6
+                    some += size >> 3
+                else:
+                    some = 0
+                self.allocated = size + some
+                byte_size = self.allocated * self.itemsize
+                if zero:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True, zero=True)
+                else:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True)
+                    copy_bytes = min(size, self.len) * self.itemsize
+                    rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer),
+                                  rffi.cast(rffi.VOIDP, self._buffer),
+                                  copy_bytes)
+            else:
+                self.len = size
+                return
+        else:
+            assert size == 0
+            self.allocated = 0
+            new_buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+        self._buffer = new_buffer
+        self.len = size
+
+    def _fromiterable(self, w_seq):
+        # used by fromsequence().
+        # a more careful case if w_seq happens to be a very large
+        # iterable: don't copy the items into some intermediate list
+        w_iterator = self.space.iter(w_seq)
+        tp = self.space.type(w_iterator)
+        while True:
+            unpack_driver.jit_merge_point(selfclass=self.__class__,
+                                          tp=tp, self=self,
+                                          w_iterator=w_iterator)
+            space = self.space
+            try:
+                w_item = space.next(w_iterator)
+            except OperationError as e:
+                if not e.match(space, space.w_StopIteration):
+                    raise
+                break  # done
+            self.descr_append(space, w_item)
+
+    def _charbuf_start(self):
+        return self._buffer
+
+    def _buffer_as_unsigned(self):
+        return rffi.cast(lltype.Unsigned, self._buffer)
+
+    def _charbuf_stop(self):
+        keepalive_until_here(self)
+
+    def delitem(self, space, i, j):
+        if i < 0:
+            i += self.len
+        if i < 0:
+            i = 0
+        if j < 0:
+            j += self.len
+        if j < 0:
+            j = 0
+        if j > self.len:
+            j = self.len
+        if i >= j:
+            return None
+        oldbuffer = self._buffer
+        self._buffer = lltype.malloc(rffi.CCHARP.TO,
+            (self.len - (j - i)) * self.itemsize, flavor='raw',
+            add_memory_pressure=True)
+        if i:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, self._buffer),
+                rffi.cast(rffi.VOIDP, oldbuffer),
+                i * self.itemsize
+            )
+        if j < self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                                  i * self.itemsize)),
+                rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer,
+                                                  j * self.itemsize)),
+                (self.len - j) * self.itemsize
+            )
+        self.len -= j - i
+        self.allocated = self.len
+        if oldbuffer:
+            lltype.free(oldbuffer, flavor='raw')
 
     def buffer_w_ex(self, space, flags):
         return ArrayBuffer(self, False), self.typecode, self.itemsize
@@ -170,14 +278,24 @@
 
         Return number of occurrences of x in the array.
         """
-        raise NotImplementedError
+        cnt = 0
+        for i in range(self.len):
+            # XXX jitdriver
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_val)):
+                cnt += 1
+        return space.wrap(cnt)
 
     def descr_index(self, space, w_x):
         """ index(x)
 
         Return index of first occurrence of x in the array.
         """
-        raise NotImplementedError
+        for i in range(self.len):
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_x)):
+                return space.wrap(i)
+        raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
     def descr_reverse(self, space):
         """ reverse()
@@ -191,7 +309,8 @@
 
         Remove the first occurrence of x in the array.
         """
-        raise NotImplementedError
+        w_idx = self.descr_index(space, w_val)
+        self.descr_pop(space, space.int_w(w_idx))
 
     @unwrap_spec(i=int)
     def descr_pop(self, space, i=-1):
@@ -507,16 +626,102 @@
         return space.newseqiter(self)
 
     def descr_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        a = self.constructor(space)
+        a.setlen(self.len + w_other.len, overallocate=False)
+        if self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, a._buffer),
+                rffi.cast(rffi.VOIDP, self._buffer),
+                self.len * self.itemsize
+            )
+        if w_other.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(a._buffer,
+                                             self.len * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                w_other.len * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_inplace_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        oldlen = self.len
+        otherlen = w_other.len
+        self.setlen(oldlen + otherlen)
+        if otherlen:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                             oldlen * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                otherlen * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(w_other)
+        return self
+
+    def _mul_helper(self, space, w_repeat, is_inplace):
+        try:
+            repeat = space.getindex_w(w_repeat, space.w_OverflowError)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
+        if is_inplace:
+            a = self
+            start = 1
+        else:
+            a = self.constructor(space)
+            start = 0
+        if repeat <= start:
+            if repeat <= 0:
+                a.setlen(0, overallocate=False)
+            return a
+        oldlen = self.len
+        try:
+            newlen = ovfcheck(oldlen * repeat)
+        except OverflowError:
+            raise MemoryError
+        #
+        srcbuf = self._buffer
+        srcsize = self.len * self.itemsize
+        for i in range(srcsize):
+            if srcbuf[i] != '\x00':
+                break
+        else:
+            # the source is entirely zero: initialize the target
+            # with zeroes too
+            a.setlen(newlen, zero=True, overallocate=False)
+            return a
+        #
+        a.setlen(newlen, overallocate=False)
+        srcbuf = self._buffer   # reload this, in case self is a
+        if oldlen == 1:
+            self._repeat_single_item(a, start, repeat)
+        else:
+            dstbuf = a._buffer
+            if start == 1:
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+            for r in range(start, repeat):
+                rffi.c_memcpy(rffi.cast(rffi.VOIDP, dstbuf),
+                              rffi.cast(rffi.VOIDP, srcbuf),
+                              srcsize)
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, False)
 
     def descr_inplace_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, True)
 
     def descr_radd(self, space, w_other):
         return self.descr_add(space, w_other)
@@ -600,6 +805,7 @@
         self.itemtype = itemtype
         self.bytes = rffi.sizeof(itemtype)
         self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
+        self.arrayptrtype = lltype.Ptr(self.arraytype)
         self.unwrap, _, self.convert = unwrap.partition('.')
         self.signed = signed
         self.canoverflow = canoverflow
@@ -694,22 +900,21 @@
         return self.array._charbuf_start()
 
 
+unpack_driver = jit.JitDriver(name='unpack_array',
+                              greens=['selfclass', 'tp'],
+                              reds=['self', 'w_iterator'])
+
 def make_array(mytype):
     W_ArrayBase = globals()['W_ArrayBase']
 
-    unpack_driver = jit.JitDriver(name='unpack_array',
-                                  greens=['tp'],
-                                  reds=['self', 'w_iterator'])
-
     class W_Array(W_ArrayBase):
         itemsize = mytype.bytes
         typecode = mytype.typecode
 
-        _attrs_ = ('space', 'len', 'allocated', '_lifeline_', 'buffer')
+        _attrs_ = W_ArrayBase._attrs_
 
-        def __init__(self, space):
-            W_ArrayBase.__init__(self, space)
-            self.buffer = lltype.nullptr(mytype.arraytype)
+        def get_buffer(self):
+            return rffi.cast(mytype.arrayptrtype, self._buffer)
 
         def item_w(self, w_item):
             space = self.space
@@ -765,46 +970,6 @@
                                          self.space.wrap(msg))
             return result
 
-        @rgc.must_be_light_finalizer
-        def __del__(self):
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-
-        def setlen(self, size, zero=False, overallocate=True):
-            if size > 0:
-                if size > self.allocated or size < self.allocated / 2:
-                    if overallocate:
-                        if size < 9:
-                            some = 3
-                        else:
-                            some = 6
-                        some += size >> 3
-                    else:
-                        some = 0
-                    self.allocated = size + some
-                    if zero:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True, zero=True)
-                    else:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True)
-                        for i in range(min(size, self.len)):
-                            new_buffer[i] = self.buffer[i]
-                else:
-                    self.len = size
-                    return
-            else:
-                assert size == 0
-                self.allocated = 0
-                new_buffer = lltype.nullptr(mytype.arraytype)
-
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-            self.buffer = new_buffer
-            self.len = size
-
         def fromsequence(self, w_seq):
             space = self.space
             oldlen = self.len
@@ -820,20 +985,21 @@
             if lst is not None:
                 self.setlen(oldlen + len(lst))
                 try:
-                    buf = self.buffer
+                    buf = self.get_buffer()
                     for num in lst:
                         buf[newlen] = self.item_from_int_or_float(num)
                         newlen += 1
                 except OperationError:
                     self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             # this is the common case: w_seq is a list or a tuple
             lst_w = space.listview_no_unpack(w_seq)
             if lst_w is not None:
                 self.setlen(oldlen + len(lst_w))
-                buf = self.buffer
+                buf = self.get_buffer()
                 try:
                     for w_num in lst_w:
                         # note: self.item_w() might invoke arbitrary code.
@@ -844,30 +1010,14 @@
                         buf[newlen] = self.item_w(w_num)
                         newlen += 1
                 except OperationError:
-                    if buf == self.buffer:
+                    if buf == self.get_buffer():
                         self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             self._fromiterable(w_seq)
 
-        def _fromiterable(self, w_seq):
-            # a more careful case if w_seq happens to be a very large
-            # iterable: don't copy the items into some intermediate list
-            w_iterator = self.space.iter(w_seq)
-            tp = self.space.type(w_iterator)
-            while True:
-                unpack_driver.jit_merge_point(tp=tp, self=self,
-                                              w_iterator=w_iterator)
-                space = self.space
-                try:
-                    w_item = space.next(w_iterator)
-                except OperationError as e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    break  # done
-                self.descr_append(space, w_item)
-
         def extend(self, w_iterable, accept_different_array=False):
             space = self.space
             if isinstance(w_iterable, W_Array):
@@ -875,11 +1025,14 @@
                 new = w_iterable.len
                 self.setlen(self.len + new)
                 i = 0
+                buf = self.get_buffer()
+                srcbuf = w_iterable.get_buffer()
                 while i < new:
                     if oldlen + i >= self.len:
                         self.setlen(oldlen + i + 1)
-                    self.buffer[oldlen + i] = w_iterable.buffer[i]
+                    buf[oldlen + i] = srcbuf[i]
                     i += 1
+                keepalive_until_here(w_iterable)
                 self.setlen(oldlen + i)
             elif (not accept_different_array
                   and isinstance(w_iterable, W_ArrayBase)):
@@ -888,17 +1041,9 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def _charbuf_start(self):
-            return rffi.cast(rffi.CCHARP, self.buffer)
-
-        def _buffer_as_unsigned(self):
-            return rffi.cast(lltype.Unsigned, self.buffer)
-
-        def _charbuf_stop(self):
+        def w_getitem(self, space, idx):
+            item = self.get_buffer()[idx]
             keepalive_until_here(self)
-
-        def w_getitem(self, space, idx):
-            item = self.buffer[idx]
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
             elif mytype.typecode == 'f':
@@ -916,29 +1061,16 @@
             x = self.item_w(w_x)
             index = self.len
             self.setlen(index + 1)
-            self.buffer[index] = x
+            self.get_buffer()[index] = x
+            keepalive_until_here(self)
 
         # List interface
-        def descr_count(self, space, w_val):
-            cnt = 0
-            for i in range(self.len):
-                # XXX jitdriver
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    cnt += 1
-            return space.wrap(cnt)
-
-        def descr_index(self, space, w_val):
-            for i in range(self.len):
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    return space.wrap(i)
-            raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
         def descr_reverse(self, space):
-            b = self.buffer
+            b = self.get_buffer()
             for i in range(self.len / 2):
                 b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i]
+            keepalive_until_here(self)
 
         def descr_pop(self, space, i):
             if i < 0:
@@ -946,16 +1078,14 @@
             if i < 0 or i >= self.len:
                 raise oefmt(space.w_IndexError, "pop index out of range")
             w_val = self.w_getitem(space, i)
+            b = self.get_buffer()
             while i < self.len - 1:
-                self.buffer[i] = self.buffer[i + 1]
+                b[i] = b[i + 1]
                 i += 1
+            keepalive_until_here(self)
             self.setlen(self.len - 1)
             return w_val
 
-        def descr_remove(self, space, w_val):
-            w_idx = self.descr_index(space, w_val)
-            self.descr_pop(space, space.int_w(w_idx))
-
         def descr_insert(self, space, idx, w_val):
             if idx < 0:
                 idx += self.len
@@ -967,10 +1097,12 @@
             val = self.item_w(w_val)
             self.setlen(self.len + 1)
             i = self.len - 1
+            b = self.get_buffer()
             while i > idx:
-                self.buffer[i] = self.buffer[i - 1]
+                b[i] = b[i - 1]
                 i -= 1
-            self.buffer[i] = val
+            b[i] = val
+            keepalive_until_here(self)
 
         def getitem_slice(self, space, w_idx):
             start, stop, step, size = space.decode_index4(w_idx, self.len)
@@ -978,9 +1110,13 @@
             w_a.setlen(size, overallocate=False)
             assert step != 0
             j = 0
+            buf = w_a.get_buffer()
+            srcbuf = self.get_buffer()
             for i in range(start, stop, step):
-                w_a.buffer[j] = self.buffer[i]
+                buf[j] = srcbuf[i]
                 j += 1
+            keepalive_until_here(self)
+            keepalive_until_here(w_a)
             return w_a
 
         def setitem(self, space, w_idx, w_item):
@@ -989,7 +1125,8 @@
                 raise oefmt(self.space.w_TypeError,
                             "can only assign array to array slice")
             item = self.item_w(w_item)
-            self.buffer[idx] = item
+            self.get_buffer()[idx] = item
+            keepalive_until_here(self)
 
         def setitem_slice(self, space, w_idx, w_item):
             if not isinstance(w_item, W_Array):
@@ -1006,127 +1143,21 @@
                 self.fromsequence(w_lst)
             else:
                 j = 0
+                buf = self.get_buffer()
+                srcbuf = w_item.get_buffer()
                 for i in range(start, stop, step):
-                    self.buffer[i] = w_item.buffer[j]
+                    buf[i] = srcbuf[j]
                     j += 1
+                keepalive_until_here(w_item)
+                keepalive_until_here(self)
 
-        def delitem(self, space, i, j):
-            if i < 0:
-                i += self.len
-            if i < 0:
-                i = 0
-            if j < 0:
-                j += self.len
-            if j < 0:
-                j = 0
-            if j > self.len:
-                j = self.len
-            if i >= j:
-                return None
-            oldbuffer = self.buffer
-            self.buffer = lltype.malloc(
-                mytype.arraytype, max(self.len - (j - i), 0), flavor='raw',
-                add_memory_pressure=True)
-            if i:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, self.buffer),
-                    rffi.cast(rffi.VOIDP, oldbuffer),
-                    i * mytype.bytes
-                )
-            if j < self.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, i)),
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer, j)),
-                    (self.len - j) * mytype.bytes
-                )
-            self.len -= j - i
-            self.allocated = self.len
-            if oldbuffer:
-                lltype.free(oldbuffer, flavor='raw')
-
-        # Add and mul methods
-        def descr_add(self, space, w_other):
-            if not isinstance(w_other, W_Array):
-                return space.w_NotImplemented
-            a = mytype.w_class(space)
-            a.setlen(self.len + w_other.len, overallocate=False)
-            if self.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, a.buffer),
-                    rffi.cast(rffi.VOIDP, self.buffer),
-                    self.len * mytype.bytes
-                )
-            if w_other.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(a.buffer, self.len)),
-                    rffi.cast(rffi.VOIDP, w_other.buffer),
-                    w_other.len * mytype.bytes
-                )
-            return a
-
-        def descr_inplace_add(self, space, w_other):
-            if not isinstance(w_other, W_Array):
-                return space.w_NotImplemented
-            oldlen = self.len
-            otherlen = w_other.len
-            self.setlen(oldlen + otherlen)
-            if otherlen:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, oldlen)),
-                    rffi.cast(rffi.VOIDP, w_other.buffer),
-                    otherlen * mytype.bytes
-                )
-            return self
-
-        def descr_mul(self, space, w_repeat):
-            return _mul_helper(space, self, w_repeat, False)
-
-        def descr_inplace_mul(self, space, w_repeat):
-            return _mul_helper(space, self, w_repeat, True)
-
-    def _mul_helper(space, self, w_repeat, is_inplace):
-        try:
-            repeat = space.getindex_w(w_repeat, space.w_OverflowError)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        repeat = max(repeat, 0)
-        try:
-            newlen = ovfcheck(self.len * repeat)
-        except OverflowError:
-            raise MemoryError
-        oldlen = self.len
-        if is_inplace:
-            a = self
-            start = 1
-        else:
-            a = mytype.w_class(space)
-            start = 0
-        # <a performance hack>
-        if oldlen == 1:
-            if mytype.unwrap == 'str_w' or mytype.unwrap == 'unicode_w':
-                zero = not ord(self.buffer[0])
-            elif mytype.unwrap == 'int_w' or mytype.unwrap == 'bigint_w':
-                zero = not widen(self.buffer[0])
-            #elif mytype.unwrap == 'float_w':
-            #    value = ...float(self.buffer[0])  xxx handle the case of -0.0
-            else:
-                zero = False
-            if zero:
-                a.setlen(newlen, zero=True, overallocate=False)
-                return a
-            a.setlen(newlen, overallocate=False)
-            item = self.buffer[0]
+        def _repeat_single_item(self, a, start, repeat):
+            # <a performance hack>
+            assert isinstance(a, W_Array)
+            item = self.get_buffer()[0]
+            dstbuf = a.get_buffer()
             for r in range(start, repeat):
-                a.buffer[r] = item
-            return a
-        # </a performance hack>
-        a.setlen(newlen, overallocate=False)
-        for r in range(start, repeat):
-            for i in range(oldlen):
-                a.buffer[r * oldlen + i] = self.buffer[i]
-        return a
+                dstbuf[r] = item
 
     mytype.w_class = W_Array
     W_Array.constructor = W_Array
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -732,7 +732,8 @@
     TY_FLOAT        = 0x06
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
-        if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
+        if (self.must_compile(deadframe, metainterp_sd, jitdriver_sd)
+                and not rstack.stack_almost_full()):
             self.start_compiling()
             try:
                 self._trace_and_compile_from_bridge(deadframe, metainterp_sd,
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -11,6 +11,7 @@
 from rpython.rlib.objectmodel import specialize, we_are_translated, r_dict
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib import rstack
 from rpython.rtyper.annlowlevel import (hlstr, cast_base_ptr_to_instance,
     cast_object_to_ptr)
 from rpython.rtyper.lltypesystem import lltype, llmemory, rstr, rffi
@@ -415,6 +416,8 @@
             if not confirm_enter_jit(*args):
                 return
             jitcounter.decay_all_counters()
+            if rstack.stack_almost_full():
+                return
             # start tracing
             from rpython.jit.metainterp.pyjitpl import MetaInterp
             metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -26,7 +26,6 @@
 
 class AsmGcRootFrameworkGCTransformer(BaseFrameworkGCTransformer):
     _asmgcc_save_restore_arguments = None
-    _seen_gctransformer_hint_close_stack = None
 
     def push_roots(self, hop, keep_current_args=False):
         livevars = self.get_livevars_for_roots(hop, keep_current_args)
@@ -50,29 +49,28 @@
         hop.genop("direct_call", [c_asm_nocollect, name])
 
     def gct_direct_call(self, hop):
+        # just a sanity check: if we find a fnptr with the hint on the
+        # _callable, then we'd also find the hint by looking only at the
+        # graph.  We'll actually change this graph only later, in
+        # start_transforming_graph().
         fnptr = hop.spaceop.args[0].value
         try:
             close_stack = fnptr._obj._callable._gctransformer_hint_close_stack_
         except AttributeError:
+            pass
+        else:
+            assert fnptr._obj.graph.func is fnptr._obj._callable
+        BaseFrameworkGCTransformer.gct_direct_call(self, hop)
+
+    def start_transforming_graph(self, graph):
+        try:
+            close_stack = graph.func._gctransformer_hint_close_stack_
+        except AttributeError:
             close_stack = False
         if close_stack:
-            self.handle_call_with_close_stack(hop)
-        else:
-            BaseFrameworkGCTransformer.gct_direct_call(self, hop)
+            self._transform_hint_close_stack(graph)
 
-    def handle_call_with_close_stack(self, hop):
-        fnptr = hop.spaceop.args[0].value
-        if self._seen_gctransformer_hint_close_stack is None:
-            self._seen_gctransformer_hint_close_stack = {}
-        if fnptr._obj.graph not in self._seen_gctransformer_hint_close_stack:
-            self._transform_hint_close_stack(fnptr)
-            self._seen_gctransformer_hint_close_stack[fnptr._obj.graph] = True
-        #
-        livevars = self.push_roots(hop)
-        self.default(hop)
-        self.pop_roots(hop, livevars)
-
-    def _transform_hint_close_stack(self, fnptr):
+    def _transform_hint_close_stack(self, graph):
         # We cannot easily pass variable amount of arguments of the call
         # across the call to the pypy_asm_stackwalk helper.  So we store
         # them away and restore them.  More precisely, we need to
@@ -83,8 +81,8 @@
         sradict = self._asmgcc_save_restore_arguments
         sra = []     # list of pointers to raw-malloced containers for args
         seen = {}
-        FUNC1 = lltype.typeOf(fnptr).TO
-        for TYPE in FUNC1.ARGS:
+        ARGS = [v.concretetype for v in graph.getargs()]
+        for TYPE in ARGS:
             if isinstance(TYPE, lltype.Ptr):
                 TYPE = llmemory.Address
             num = seen.get(TYPE, 0)
@@ -98,8 +96,10 @@
             sra.append(sradict[key])
         #
         # make a copy of the graph that will reload the values
-        graph = fnptr._obj.graph
         graph2 = copygraph(graph)
+        del graph2.func   # otherwise, start_transforming_graph() will
+                          # again transform graph2, and we get an
+                          # infinite loop
         #
         # edit the original graph to only store the value of the arguments
         block = Block(graph.startblock.inputargs)
@@ -116,17 +116,18 @@
                 SpaceOperation("bare_setfield", [c_p, c_item0, v_arg], v_void))
         #
         # call asm_stackwalk(graph2)
-        FUNC2 = lltype.FuncType([], FUNC1.RESULT)
+        RESULT = graph.getreturnvar().concretetype
+        FUNC2 = lltype.FuncType([], RESULT)
         fnptr2 = lltype.functionptr(FUNC2,
-                                    fnptr._obj._name + '_reload',
+                                    graph.name + '_reload',
                                     graph=graph2)
         c_fnptr2 = Constant(fnptr2, lltype.Ptr(FUNC2))
         HELPERFUNC = lltype.FuncType([lltype.Ptr(FUNC2),
-                                      ASM_FRAMEDATA_HEAD_PTR], FUNC1.RESULT)
+                                      ASM_FRAMEDATA_HEAD_PTR], RESULT)
         v_asm_stackwalk = varoftype(lltype.Ptr(HELPERFUNC), "asm_stackwalk")
         block.operations.append(
             SpaceOperation("cast_pointer", [c_asm_stackwalk], v_asm_stackwalk))
-        v_result = varoftype(FUNC1.RESULT)
+        v_result = varoftype(RESULT)
         block.operations.append(
             SpaceOperation("indirect_call", [v_asm_stackwalk, c_fnptr2,
                                              c_gcrootanchor,
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -201,6 +201,9 @@
         self.var_last_needed_in = None
         self.curr_block = None
 
+    def start_transforming_graph(self, graph):
+        pass    # for asmgcc.py
+
     def transform_graph(self, graph):
         if graph in self.minimal_transform:
             if self.minimalgctransformer:
@@ -210,6 +213,7 @@
         if graph in self.seen_graphs:
             return
         self.seen_graphs.add(graph)
+        self.start_transforming_graph(graph)
 
         self.links_to_split = {} # link -> vars to pop_alive across the link
 
diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py
--- a/rpython/rlib/rstack.py
+++ b/rpython/rlib/rstack.py
@@ -69,3 +69,22 @@
         raise _StackOverflow
 stack_check_slowpath._dont_inline_ = True
 stack_check_slowpath._dont_insert_stackcheck_ = True
+
+def stack_almost_full():
+    """Return True if the stack is more than 15/16th full."""
+    if not we_are_translated():
+        return False
+    # see stack_check()
+    current = llop.stack_current(lltype.Signed)
+    end = _stack_get_end()
+    length = 15 * (r_uint(_stack_get_length()) >> 4)
+    ofs = r_uint(end - current)
+    if ofs <= length:
+        return False    # fine
+    else:
+        _stack_too_big_slowpath(current)   # this might update the stack end
+        end = _stack_get_end()
+        ofs = r_uint(end - current)
+        return ofs > length
+stack_almost_full._dont_insert_stackcheck_ = True
+stack_almost_full._jit_look_inside_ = False
diff --git a/rpython/rlib/test/test_rsignal.py b/rpython/rlib/test/test_rsignal.py
--- a/rpython/rlib/test/test_rsignal.py
+++ b/rpython/rlib/test/test_rsignal.py
@@ -1,4 +1,4 @@
-import os, py
+import os, py, errno
 from rpython.translator.c.test.test_genc import compile
 from rpython.rlib import rsignal
 
@@ -37,3 +37,24 @@
 def test_compile():
     fn = compile(test_simple, [])
     fn()
+
+def test_compile_wakeup_fd():
+    def fn():
+        rd, wr = os.pipe()
+        rsignal.pypysig_set_wakeup_fd(wr, False)
+        for i in range(3):
+            rsignal.pypysig_setflag(rsignal.SIGUSR1)
+            os.kill(os.getpid(), rsignal.SIGUSR1)
+            check(rsignal.SIGUSR1)
+            check(-1)
+            check(-1)
+        x = os.read(rd, 10)
+        assert x == chr(rsignal.SIGUSR1) * 3
+        #
+        rsignal.pypysig_set_wakeup_fd(rd, False)   # can't write there
+        os.kill(os.getpid(), rsignal.SIGUSR1)
+
+    fn = compile(fn, [], return_stderr=True)
+    stderr = fn()
+    assert stderr.endswith('Exception ignored when trying to write to the '
+                           'signal wakeup fd: Errno %d\n' % errno.EBADF)
diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c
--- a/rpython/translator/c/src/signals.c
+++ b/rpython/translator/c/src/signals.c
@@ -2,6 +2,7 @@
 
 #include <limits.h>
 #include <stdlib.h>
+#include <errno.h>
 #ifdef _WIN32
 #include <process.h>
 #include <io.h>
@@ -82,27 +83,51 @@
       }
 }
 
+static void write_str(int fd, const char *p)
+{
+    int i = 0;
+    while (p[i] != '\x00')
+        i++;
+    (void)write(fd, p, i);
+}
+
 static void signal_setflag_handler(int signum)
 {
     pypysig_pushback(signum);
 
-    if (wakeup_fd != -1) 
-      {
+    /* Warning, this logic needs to be async-signal-safe */
+    if (wakeup_fd != -1) {
 #ifndef _WIN32
         ssize_t res;
 #else
         int res;
 #endif
-	if (wakeup_with_nul_byte)
-	{
-	    res = write(wakeup_fd, "\0", 1);
-	} else {
-	    unsigned char byte = (unsigned char)signum;
-	    res = write(wakeup_fd, &byte, 1);
-	}
-
-        /* the return value is ignored here */
-      }
+        int old_errno = errno;
+     retry:
+        if (wakeup_with_nul_byte) {
+            res = write(wakeup_fd, "\0", 1);
+        } else {
+            unsigned char byte = (unsigned char)signum;
+            res = write(wakeup_fd, &byte, 1);
+        }
+        if (res < 0) {
+            unsigned int e = (unsigned int)errno;
+            char c[27], *p;
+            if (e == EINTR)
+                goto retry;
+            write_str(2, "Exception ignored when trying to write to the "
+                         "signal wakeup fd: Errno ");
+            p = c + sizeof(c);
+            *--p = 0;
+            *--p = '\n';
+            do {
+                *--p = '0' + e % 10;
+                e /= 10;
+            } while (e != 0);
+            write_str(2, p);
+        }
+        errno = old_errno;
+    }
 }
 
 void pypysig_setflag(int signum)

From pypy.commits at gmail.com  Thu Feb  2 10:18:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 02 Feb 2017 07:18:16 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in timfel/pypy (pull request
 #512)
Message-ID: <58934db8.2ea6df0a.74481.f799@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r89902:866867e47a45
Date: 2017-02-02 15:17 +0000
http://bitbucket.org/pypy/pypy/changeset/866867e47a45/

Log:	Merged in timfel/pypy (pull request #512)

	Make attach_gdb work on Windows (with Visual Studio Debugger)

diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -446,8 +446,85 @@
             time.sleep(1)  # give the GDB time to attach
 
 else:
+    def make_vs_attach_eci():
+        # The COM interface to the Debugger has to be compiled as a .cpp file by
+        # Visual C. So we generate the source and then add a commandline switch
+        # to treat this source file as C++
+        import os
+        eci = ExternalCompilationInfo(post_include_bits=["""
+#ifdef __cplusplus
+extern "C" {
+#endif
+RPY_EXPORTED void AttachToVS();
+#ifdef __cplusplus
+}
+#endif
+                                      """],
+                                      separate_module_sources=["""
+#import "libid:80cc9f66-e7d8-4ddd-85b6-d9e6cd0e93e2" version("8.0") lcid("0") raw_interfaces_only named_guids
+extern "C" RPY_EXPORTED void AttachToVS() {
+    CoInitialize(0);
+    HRESULT hr;
+    CLSID Clsid;
+
+    CLSIDFromProgID(L"VisualStudio.DTE", &Clsid);
+    IUnknown *Unknown;
+    if (FAILED(GetActiveObject(Clsid, 0, &Unknown))) {
+        puts("Could not attach to Visual Studio (is it not running?");
+        return;
+    }
+
+    EnvDTE::_DTE *Interface;
+    hr = Unknown->QueryInterface(&Interface);
+    if (FAILED(GetActiveObject(Clsid, 0, &Unknown))) {
+        puts("Could not open COM interface to Visual Studio (no permissions?)");
+        return;
+    }
+
+    EnvDTE::Debugger *Debugger;
+    puts("Waiting for Visual Studio Debugger to become idle");
+    while (FAILED(Interface->get_Debugger(&Debugger)));
+
+    EnvDTE::Processes *Processes;
+    while (FAILED(Debugger->get_LocalProcesses(&Processes)));
+
+    long Count = 0;
+    if (FAILED(Processes->get_Count(&Count))) {
+        puts("Cannot query Process count");
+    }
+
+    for (int i = 0; i <= Count; i++) {
+        EnvDTE::Process *Process;
+        if (FAILED(Processes->Item(variant_t(i), &Process))) {
+            continue;
+        }
+
+        long ProcessID;
+        while (FAILED(Process->get_ProcessID(&ProcessID)));
+
+        if (ProcessID == GetProcessId(GetCurrentProcess())) {
+            printf("Found process ID %d\\n", ProcessID);
+            Process->Attach();
+            Debugger->Break(false);
+            CoUninitialize();
+            return;
+        }
+    }
+}
+                                      """]
+        )
+        eci = eci.convert_sources_to_files()
+        d = eci._copy_attributes()
+        cfile = d['separate_module_files'][0]
+        cppfile = cfile.replace(".c", "_vsdebug.cpp")
+        os.rename(cfile, cppfile)
+        d['separate_module_files'] = [cppfile]
+        return ExternalCompilationInfo(**d)
+
+    ll_attach = rffi.llexternal("AttachToVS", [], lltype.Void,
+                                compilation_info=make_vs_attach_eci())
     def impl_attach_gdb():
-        print "Don't know how to attach GDB on Windows"
+        ll_attach()
 
 register_external(attach_gdb, [], result=None,
                   export_name="impl_attach_gdb", llimpl=impl_attach_gdb)
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -379,9 +379,9 @@
             no_precompile = []
             for f in list(no_precompile_cfiles):
                 f = m.pathrel(py.path.local(f))
-                if f not in no_precompile and f.endswith('.c'):
+                if f not in no_precompile and (f.endswith('.c') or f.endswith('.cpp')):
                     no_precompile.append(f)
-                    target = f[:-1] + 'obj'
+                    target = f[:f.rfind('.')] + '.obj'
                     rules.append((target, f,
                         '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
                         '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f)))

From pypy.commits at gmail.com  Thu Feb  2 10:18:24 2017
From: pypy.commits at gmail.com (timfel)
Date: Thu, 02 Feb 2017 07:18:24 -0800 (PST)
Subject: [pypy-commit] pypy default: make attach_gdb work on Windows (with
 Visual Studio Debugger)
Message-ID: <58934dc0.484a1c0a.4736.434f@mx.google.com>

Author: Tim Felgentreff <timfelgentreff at gmail.com>
Branch: 
Changeset: r89900:a64ec5d7ebcb
Date: 2017-02-02 11:07 +0100
http://bitbucket.org/pypy/pypy/changeset/a64ec5d7ebcb/

Log:	make attach_gdb work on Windows (with Visual Studio Debugger)

diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -446,8 +446,85 @@
             time.sleep(1)  # give the GDB time to attach
 
 else:
+    def make_vs_attach_eci():
+        # The COM interface to the Debugger has to be compiled as a .cpp file by
+        # Visual C. So we generate the source and then add a commandline switch
+        # to treat this source file as C++
+        import os
+        eci = ExternalCompilationInfo(post_include_bits=["""
+#ifdef __cplusplus
+extern "C" {
+#endif
+RPY_EXPORTED void AttachToVS();
+#ifdef __cplusplus
+}
+#endif
+                                      """],
+                                      separate_module_sources=["""
+#import "libid:80cc9f66-e7d8-4ddd-85b6-d9e6cd0e93e2" version("8.0") lcid("0") raw_interfaces_only named_guids
+extern "C" RPY_EXPORTED void AttachToVS() {
+    CoInitialize(0);
+    HRESULT hr;
+    CLSID Clsid;
+
+    CLSIDFromProgID(L"VisualStudio.DTE", &Clsid);
+    IUnknown *Unknown;
+    if (FAILED(GetActiveObject(Clsid, 0, &Unknown))) {
+        puts("Could not attach to Visual Studio (is it not running?");
+        return;
+    }
+
+    EnvDTE::_DTE *Interface;
+    hr = Unknown->QueryInterface(&Interface);
+    if (FAILED(GetActiveObject(Clsid, 0, &Unknown))) {
+        puts("Could not open COM interface to Visual Studio (no permissions?)");
+        return;
+    }
+
+    EnvDTE::Debugger *Debugger;
+    puts("Waiting for Visual Studio Debugger to become idle");
+    while (FAILED(Interface->get_Debugger(&Debugger)));
+
+    EnvDTE::Processes *Processes;
+    while (FAILED(Debugger->get_LocalProcesses(&Processes)));
+
+    long Count = 0;
+    if (FAILED(Processes->get_Count(&Count))) {
+        puts("Cannot query Process count");
+    }
+
+    for (int i = 0; i <= Count; i++) {
+        EnvDTE::Process *Process;
+        if (FAILED(Processes->Item(variant_t(i), &Process))) {
+            continue;
+        }
+
+        long ProcessID;
+        while (FAILED(Process->get_ProcessID(&ProcessID)));
+
+        if (ProcessID == GetProcessId(GetCurrentProcess())) {
+            printf("Found process ID %d\\n", ProcessID);
+            Process->Attach();
+            Debugger->Break(false);
+            CoUninitialize();
+            return;
+        }
+    }
+}
+                                      """]
+        )
+        eci = eci.convert_sources_to_files()
+        d = eci._copy_attributes()
+        cfile = d['separate_module_files'][0]
+        cppfile = cfile.replace(".c", "_vsdebug.cpp")
+        os.rename(cfile, cppfile)
+        d['separate_module_files'] = [cppfile]
+        return ExternalCompilationInfo(**d)
+
+    ll_attach = rffi.llexternal("AttachToVS", [], lltype.Void,
+                                compilation_info=make_vs_attach_eci())
     def impl_attach_gdb():
-        print "Don't know how to attach GDB on Windows"
+        ll_attach()
 
 register_external(attach_gdb, [], result=None,
                   export_name="impl_attach_gdb", llimpl=impl_attach_gdb)
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -379,9 +379,9 @@
             no_precompile = []
             for f in list(no_precompile_cfiles):
                 f = m.pathrel(py.path.local(f))
-                if f not in no_precompile and f.endswith('.c'):
+                if f not in no_precompile and (f.endswith('.c') or f.endswith('.cpp')):
                     no_precompile.append(f)
-                    target = f[:-1] + 'obj'
+                    target = f[:f.rfind('.')] + '.obj'
                     rules.append((target, f,
                         '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
                         '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f)))

From pypy.commits at gmail.com  Thu Feb  2 10:18:28 2017
From: pypy.commits at gmail.com (timfel)
Date: Thu, 02 Feb 2017 07:18:28 -0800 (PST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <58934dc4.125e1c0a.76a00.14f1@mx.google.com>

Author: Tim Felgentreff <timfelgentreff at gmail.com>
Branch: 
Changeset: r89901:c613943ef39a
Date: 2017-02-02 11:07 +0100
http://bitbucket.org/pypy/pypy/changeset/c613943ef39a/

Log:	merge default

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -102,7 +102,11 @@
                 % (len(buf) + offset, size + offset))
         raw_addr = buf._pypy_raw_address()
         result = self.from_address(raw_addr)
-        result._ensure_objects()['ffffffff'] = obj
+        objects = result._ensure_objects()
+        if objects is not None:
+            objects['ffffffff'] = obj
+        else:   # case e.g. of a primitive type like c_int
+            result._objects = obj
         return result
 
     def from_buffer_copy(self, obj, offset=0):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -198,10 +198,21 @@
 
     def getdict(self, space):
         if self.w_func_dict is None:
+            if not self.can_change_code:
+                raise oefmt(space.w_AttributeError,
+                            "cannot set extra attributes on built-in functions")
             self.w_func_dict = space.newdict(instance=True)
         return self.w_func_dict
 
+    def getdictvalue(self, space, attr):
+        if not self.can_change_code:
+            return None
+        return W_Root.getdictvalue(self, space, attr)
+
     def setdict(self, space, w_dict):
+        if not self.can_change_code:
+            raise oefmt(space.w_AttributeError,
+                        "cannot set __dict__ on built-in functions")
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError,
                         "setting function's dictionary to a non-dict")
@@ -660,7 +671,7 @@
         Function.__init__(self, func.space, func.code, func.w_func_globals,
                           func.defs_w, func.closure, func.name)
         self.w_doc = func.w_doc
-        self.w_func_dict = func.w_func_dict
+        #self.w_func_dict = func.w_func_dict---nowadays, always None
         self.w_module = func.w_module
 
     def descr_builtinfunction__new__(space, w_subtype):
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -95,8 +95,16 @@
     def test_write_code_builtin_forbidden(self):
         def f(*args):
             return 42
-            raises(TypeError, "dir.func_code = f.func_code")
-            raises(TypeError, "list.append.im_func.func_code = f.func_code")
+        raises(TypeError, "dir.func_code = f.func_code")
+        raises(TypeError, "list.append.im_func.func_code = f.func_code")
+
+    def test_write_extra_attributes_builtin_forbidden(self):
+        raises(AttributeError, "dir.abcd = 5")
+        raises(AttributeError, "list.append.im_func.efgh = 6")
+        raises(AttributeError, "dir.__dict__")
+        raises(AttributeError, "dir.__dict__ = {}")
+        c = all.__call__   # this should work
+        assert c([4, 5, 6]) is True
 
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -122,12 +122,120 @@
                                         hints={'nolength': True}))
 
 class W_ArrayBase(W_Root):
-    _attrs_ = ('space', 'len', 'allocated', '_lifeline_') # no buffer
+    _attrs_ = ('space', 'len', 'allocated', '_lifeline_', '_buffer')
 
     def __init__(self, space):
         self.space = space
         self.len = 0
         self.allocated = 0
+        self._buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+
+    def setlen(self, size, zero=False, overallocate=True):
+        if size > 0:
+            if size > self.allocated or size < self.allocated / 2:
+                if overallocate:
+                    if size < 9:
+                        some = 3
+                    else:
+                        some = 6
+                    some += size >> 3
+                else:
+                    some = 0
+                self.allocated = size + some
+                byte_size = self.allocated * self.itemsize
+                if zero:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True, zero=True)
+                else:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True)
+                    copy_bytes = min(size, self.len) * self.itemsize
+                    rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer),
+                                  rffi.cast(rffi.VOIDP, self._buffer),
+                                  copy_bytes)
+            else:
+                self.len = size
+                return
+        else:
+            assert size == 0
+            self.allocated = 0
+            new_buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+        self._buffer = new_buffer
+        self.len = size
+
+    def _fromiterable(self, w_seq):
+        # used by fromsequence().
+        # a more careful case if w_seq happens to be a very large
+        # iterable: don't copy the items into some intermediate list
+        w_iterator = self.space.iter(w_seq)
+        tp = self.space.type(w_iterator)
+        while True:
+            unpack_driver.jit_merge_point(selfclass=self.__class__,
+                                          tp=tp, self=self,
+                                          w_iterator=w_iterator)
+            space = self.space
+            try:
+                w_item = space.next(w_iterator)
+            except OperationError as e:
+                if not e.match(space, space.w_StopIteration):
+                    raise
+                break  # done
+            self.descr_append(space, w_item)
+
+    def _charbuf_start(self):
+        return self._buffer
+
+    def _buffer_as_unsigned(self):
+        return rffi.cast(lltype.Unsigned, self._buffer)
+
+    def _charbuf_stop(self):
+        keepalive_until_here(self)
+
+    def delitem(self, space, i, j):
+        if i < 0:
+            i += self.len
+        if i < 0:
+            i = 0
+        if j < 0:
+            j += self.len
+        if j < 0:
+            j = 0
+        if j > self.len:
+            j = self.len
+        if i >= j:
+            return None
+        oldbuffer = self._buffer
+        self._buffer = lltype.malloc(rffi.CCHARP.TO,
+            (self.len - (j - i)) * self.itemsize, flavor='raw',
+            add_memory_pressure=True)
+        if i:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, self._buffer),
+                rffi.cast(rffi.VOIDP, oldbuffer),
+                i * self.itemsize
+            )
+        if j < self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                                  i * self.itemsize)),
+                rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer,
+                                                  j * self.itemsize)),
+                (self.len - j) * self.itemsize
+            )
+        self.len -= j - i
+        self.allocated = self.len
+        if oldbuffer:
+            lltype.free(oldbuffer, flavor='raw')
 
     def readbuf_w(self, space):
         return ArrayBuffer(self, True)
@@ -154,14 +262,24 @@
 
         Return number of occurrences of x in the array.
         """
-        raise NotImplementedError
+        cnt = 0
+        for i in range(self.len):
+            # XXX jitdriver
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_val)):
+                cnt += 1
+        return space.wrap(cnt)
 
     def descr_index(self, space, w_x):
         """ index(x)
 
         Return index of first occurrence of x in the array.
         """
-        raise NotImplementedError
+        for i in range(self.len):
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_x)):
+                return space.wrap(i)
+        raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
     def descr_reverse(self, space):
         """ reverse()
@@ -175,7 +293,8 @@
 
         Remove the first occurrence of x in the array.
         """
-        raise NotImplementedError
+        w_idx = self.descr_index(space, w_val)
+        self.descr_pop(space, space.int_w(w_idx))
 
     @unwrap_spec(i=int)
     def descr_pop(self, space, i=-1):
@@ -453,16 +572,102 @@
         return space.newseqiter(self)
 
     def descr_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        a = self.constructor(space)
+        a.setlen(self.len + w_other.len, overallocate=False)
+        if self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, a._buffer),
+                rffi.cast(rffi.VOIDP, self._buffer),
+                self.len * self.itemsize
+            )
+        if w_other.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(a._buffer,
+                                             self.len * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                w_other.len * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_inplace_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        oldlen = self.len
+        otherlen = w_other.len
+        self.setlen(oldlen + otherlen)
+        if otherlen:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                             oldlen * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                otherlen * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(w_other)
+        return self
+
+    def _mul_helper(self, space, w_repeat, is_inplace):
+        try:
+            repeat = space.getindex_w(w_repeat, space.w_OverflowError)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
+        if is_inplace:
+            a = self
+            start = 1
+        else:
+            a = self.constructor(space)
+            start = 0
+        if repeat <= start:
+            if repeat <= 0:
+                a.setlen(0, overallocate=False)
+            return a
+        oldlen = self.len
+        try:
+            newlen = ovfcheck(oldlen * repeat)
+        except OverflowError:
+            raise MemoryError
+        #
+        srcbuf = self._buffer
+        srcsize = self.len * self.itemsize
+        for i in range(srcsize):
+            if srcbuf[i] != '\x00':
+                break
+        else:
+            # the source is entirely zero: initialize the target
+            # with zeroes too
+            a.setlen(newlen, zero=True, overallocate=False)
+            return a
+        #
+        a.setlen(newlen, overallocate=False)
+        srcbuf = self._buffer   # reload this, in case self is a
+        if oldlen == 1:
+            self._repeat_single_item(a, start, repeat)
+        else:
+            dstbuf = a._buffer
+            if start == 1:
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+            for r in range(start, repeat):
+                rffi.c_memcpy(rffi.cast(rffi.VOIDP, dstbuf),
+                              rffi.cast(rffi.VOIDP, srcbuf),
+                              srcsize)
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, False)
 
     def descr_inplace_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, True)
 
     def descr_radd(self, space, w_other):
         return self.descr_add(space, w_other)
@@ -551,6 +756,7 @@
         self.itemtype = itemtype
         self.bytes = rffi.sizeof(itemtype)
         self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
+        self.arrayptrtype = lltype.Ptr(self.arraytype)
         self.unwrap = unwrap
         self.signed = signed
         self.canoverflow = canoverflow
@@ -640,22 +846,21 @@
         return self.array._charbuf_start()
 
 
+unpack_driver = jit.JitDriver(name='unpack_array',
+                              greens=['selfclass', 'tp'],
+                              reds=['self', 'w_iterator'])
+
 def make_array(mytype):
     W_ArrayBase = globals()['W_ArrayBase']
 
-    unpack_driver = jit.JitDriver(name='unpack_array',
-                                  greens=['tp'],
-                                  reds=['self', 'w_iterator'])
-
     class W_Array(W_ArrayBase):
         itemsize = mytype.bytes
         typecode = mytype.typecode
 
-        _attrs_ = ('space', 'len', 'allocated', '_lifeline_', 'buffer')
+        _attrs_ = W_ArrayBase._attrs_
 
-        def __init__(self, space):
-            W_ArrayBase.__init__(self, space)
-            self.buffer = lltype.nullptr(mytype.arraytype)
+        def get_buffer(self):
+            return rffi.cast(mytype.arrayptrtype, self._buffer)
 
         def item_w(self, w_item):
             space = self.space
@@ -709,46 +914,6 @@
                                          self.space.wrap(msg))
             return result
 
-        @rgc.must_be_light_finalizer
-        def __del__(self):
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-
-        def setlen(self, size, zero=False, overallocate=True):
-            if size > 0:
-                if size > self.allocated or size < self.allocated / 2:
-                    if overallocate:
-                        if size < 9:
-                            some = 3
-                        else:
-                            some = 6
-                        some += size >> 3
-                    else:
-                        some = 0
-                    self.allocated = size + some
-                    if zero:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True, zero=True)
-                    else:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True)
-                        for i in range(min(size, self.len)):
-                            new_buffer[i] = self.buffer[i]
-                else:
-                    self.len = size
-                    return
-            else:
-                assert size == 0
-                self.allocated = 0
-                new_buffer = lltype.nullptr(mytype.arraytype)
-
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-            self.buffer = new_buffer
-            self.len = size
-
         def fromsequence(self, w_seq):
             space = self.space
             oldlen = self.len
@@ -764,20 +929,21 @@
             if lst is not None:
                 self.setlen(oldlen + len(lst))
                 try:
-                    buf = self.buffer
+                    buf = self.get_buffer()
                     for num in lst:
                         buf[newlen] = self.item_from_int_or_float(num)
                         newlen += 1
                 except OperationError:
                     self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             # this is the common case: w_seq is a list or a tuple
             lst_w = space.listview_no_unpack(w_seq)
             if lst_w is not None:
                 self.setlen(oldlen + len(lst_w))
-                buf = self.buffer
+                buf = self.get_buffer()
                 try:
                     for w_num in lst_w:
                         # note: self.item_w() might invoke arbitrary code.
@@ -788,30 +954,14 @@
                         buf[newlen] = self.item_w(w_num)
                         newlen += 1
                 except OperationError:
-                    if buf == self.buffer:
+                    if buf == self.get_buffer():
                         self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             self._fromiterable(w_seq)
 
-        def _fromiterable(self, w_seq):
-            # a more careful case if w_seq happens to be a very large
-            # iterable: don't copy the items into some intermediate list
-            w_iterator = self.space.iter(w_seq)
-            tp = self.space.type(w_iterator)
-            while True:
-                unpack_driver.jit_merge_point(tp=tp, self=self,
-                                              w_iterator=w_iterator)
-                space = self.space
-                try:
-                    w_item = space.next(w_iterator)
-                except OperationError as e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    break  # done
-                self.descr_append(space, w_item)
-
         def extend(self, w_iterable, accept_different_array=False):
             space = self.space
             if isinstance(w_iterable, W_Array):
@@ -819,11 +969,14 @@
                 new = w_iterable.len
                 self.setlen(self.len + new)
                 i = 0
+                buf = self.get_buffer()
+                srcbuf = w_iterable.get_buffer()
                 while i < new:
                     if oldlen + i >= self.len:
                         self.setlen(oldlen + i + 1)
-                    self.buffer[oldlen + i] = w_iterable.buffer[i]
+                    buf[oldlen + i] = srcbuf[i]
                     i += 1
+                keepalive_until_here(w_iterable)
                 self.setlen(oldlen + i)
             elif (not accept_different_array
                   and isinstance(w_iterable, W_ArrayBase)):
@@ -832,17 +985,9 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def _charbuf_start(self):
-            return rffi.cast(rffi.CCHARP, self.buffer)
-
-        def _buffer_as_unsigned(self):
-            return rffi.cast(lltype.Unsigned, self.buffer)
-
-        def _charbuf_stop(self):
+        def w_getitem(self, space, idx):
+            item = self.get_buffer()[idx]
             keepalive_until_here(self)
-
-        def w_getitem(self, space, idx):
-            item = self.buffer[idx]
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
             elif mytype.typecode == 'f':
@@ -855,29 +1000,16 @@
             x = self.item_w(w_x)
             index = self.len
             self.setlen(index + 1)
-            self.buffer[index] = x
+            self.get_buffer()[index] = x
+            keepalive_until_here(self)
 
         # List interface
-        def descr_count(self, space, w_val):
-            cnt = 0
-            for i in range(self.len):
-                # XXX jitdriver
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    cnt += 1
-            return space.wrap(cnt)
-
-        def descr_index(self, space, w_val):
-            for i in range(self.len):
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    return space.wrap(i)
-            raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
         def descr_reverse(self, space):
-            b = self.buffer
+            b = self.get_buffer()
             for i in range(self.len / 2):
                 b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i]
+            keepalive_until_here(self)
 
         def descr_pop(self, space, i):
             if i < 0:
@@ -885,16 +1017,14 @@
             if i < 0 or i >= self.len:
                 raise oefmt(space.w_IndexError, "pop index out of range")
             w_val = self.w_getitem(space, i)
+            b = self.get_buffer()
             while i < self.len - 1:
-                self.buffer[i] = self.buffer[i + 1]
+                b[i] = b[i + 1]
                 i += 1
+            keepalive_until_here(self)
             self.setlen(self.len - 1)
             return w_val
 
-        def descr_remove(self, space, w_val):
-            w_idx = self.descr_index(space, w_val)
-            self.descr_pop(space, space.int_w(w_idx))
-
         def descr_insert(self, space, idx, w_val):
             if idx < 0:
                 idx += self.len
@@ -906,10 +1036,12 @@
             val = self.item_w(w_val)
             self.setlen(self.len + 1)
             i = self.len - 1
+            b = self.get_buffer()
             while i > idx:
-                self.buffer[i] = self.buffer[i - 1]
+                b[i] = b[i - 1]
                 i -= 1
-            self.buffer[i] = val
+            b[i] = val
+            keepalive_until_here(self)
 
         def getitem_slice(self, space, w_idx):
             start, stop, step, size = space.decode_index4(w_idx, self.len)
@@ -917,9 +1049,13 @@
             w_a.setlen(size, overallocate=False)
             assert step != 0
             j = 0
+            buf = w_a.get_buffer()
+            srcbuf = self.get_buffer()
             for i in range(start, stop, step):
-                w_a.buffer[j] = self.buffer[i]
+                buf[j] = srcbuf[i]
                 j += 1
+            keepalive_until_here(self)
+            keepalive_until_here(w_a)
             return w_a
 
         def setitem(self, space, w_idx, w_item):
@@ -928,7 +1064,8 @@
                 raise oefmt(self.space.w_TypeError,
                             "can only assign array to array slice")
             item = self.item_w(w_item)
-            self.buffer[idx] = item
+            self.get_buffer()[idx] = item
+            keepalive_until_here(self)
 
         def setitem_slice(self, space, w_idx, w_item):
             if not isinstance(w_item, W_Array):
@@ -945,127 +1082,21 @@
                 self.fromsequence(w_lst)
             else:
                 j = 0
+                buf = self.get_buffer()
+                srcbuf = w_item.get_buffer()
                 for i in range(start, stop, step):
-                    self.buffer[i] = w_item.buffer[j]
+                    buf[i] = srcbuf[j]
                     j += 1
+                keepalive_until_here(w_item)
+                keepalive_until_here(self)
 
-        def delitem(self, space, i, j):
-            if i < 0:
-                i += self.len
-            if i < 0:
-                i = 0
-            if j < 0:
-                j += self.len
-            if j < 0:
-                j = 0
-            if j > self.len:
-                j = self.len
-            if i >= j:
-                return None
-            oldbuffer = self.buffer
-            self.buffer = lltype.malloc(
-                mytype.arraytype, max(self.len - (j - i), 0), flavor='raw',
-                add_memory_pressure=True)
-            if i:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, self.buffer),
-                    rffi.cast(rffi.VOIDP, oldbuffer),
-                    i * mytype.bytes
-                )
-            if j < self.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, i)),
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer, j)),
-                    (self.len - j) * mytype.bytes
-                )
-            self.len -= j - i
-            self.allocated = self.len
-            if oldbuffer:
-                lltype.free(oldbuffer, flavor='raw')
-
-        # Add and mul methods
-        def descr_add(self, space, w_other):
-            if not isinstance(w_other, W_Array):
-                return space.w_NotImplemented
-            a = mytype.w_class(space)
-            a.setlen(self.len + w_other.len, overallocate=False)
-            if self.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, a.buffer),
-                    rffi.cast(rffi.VOIDP, self.buffer),
-                    self.len * mytype.bytes
-                )
-            if w_other.len:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(a.buffer, self.len)),
-                    rffi.cast(rffi.VOIDP, w_other.buffer),
-                    w_other.len * mytype.bytes
-                )
-            return a
-
-        def descr_inplace_add(self, space, w_other):
-            if not isinstance(w_other, W_Array):
-                return space.w_NotImplemented
-            oldlen = self.len
-            otherlen = w_other.len
-            self.setlen(oldlen + otherlen)
-            if otherlen:
-                rffi.c_memcpy(
-                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, oldlen)),
-                    rffi.cast(rffi.VOIDP, w_other.buffer),
-                    otherlen * mytype.bytes
-                )
-            return self
-
-        def descr_mul(self, space, w_repeat):
-            return _mul_helper(space, self, w_repeat, False)
-
-        def descr_inplace_mul(self, space, w_repeat):
-            return _mul_helper(space, self, w_repeat, True)
-
-    def _mul_helper(space, self, w_repeat, is_inplace):
-        try:
-            repeat = space.getindex_w(w_repeat, space.w_OverflowError)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        repeat = max(repeat, 0)
-        try:
-            newlen = ovfcheck(self.len * repeat)
-        except OverflowError:
-            raise MemoryError
-        oldlen = self.len
-        if is_inplace:
-            a = self
-            start = 1
-        else:
-            a = mytype.w_class(space)
-            start = 0
-        # <a performance hack>
-        if oldlen == 1:
-            if mytype.unwrap == 'str_w' or mytype.unwrap == 'unicode_w':
-                zero = not ord(self.buffer[0])
-            elif mytype.unwrap == 'int_w' or mytype.unwrap == 'bigint_w':
-                zero = not widen(self.buffer[0])
-            #elif mytype.unwrap == 'float_w':
-            #    value = ...float(self.buffer[0])  xxx handle the case of -0.0
-            else:
-                zero = False
-            if zero:
-                a.setlen(newlen, zero=True, overallocate=False)
-                return a
-            a.setlen(newlen, overallocate=False)
-            item = self.buffer[0]
+        def _repeat_single_item(self, a, start, repeat):
+            # <a performance hack>
+            assert isinstance(a, W_Array)
+            item = self.get_buffer()[0]
+            dstbuf = a.get_buffer()
             for r in range(start, repeat):
-                a.buffer[r] = item
-            return a
-        # </a performance hack>
-        a.setlen(newlen, overallocate=False)
-        for r in range(start, repeat):
-            for i in range(oldlen):
-                a.buffer[r * oldlen + i] = self.buffer[i]
-        return a
+                dstbuf[r] = item
 
     mytype.w_class = W_Array
     W_Array.constructor = W_Array
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py
@@ -24,6 +24,16 @@
         assert b[0] == "a"
         assert b[:] == "abc\0"
 
+    def test_from_buffer(self):
+        b1 = bytearray("abcde")
+        b = (c_char * 5).from_buffer(b1)
+        assert b[2] == "c"
+        #
+        b1 = bytearray("abcd")
+        b = c_int.from_buffer(b1)
+        assert b.value in (1684234849,   # little endian
+                           1633837924)   # big endian
+
     try:
         c_wchar
     except NameError:
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -281,6 +281,8 @@
 
 def test_elidable_kinds():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
+    from rpython.rlib.objectmodel import compute_hash
+    from rpython.rlib.rsiphash import enable_siphash24
 
     @jit.elidable
     def f1(n, m):
@@ -293,12 +295,17 @@
         if n > m:
             raise ValueError
         return n + m
+    @jit.elidable
+    def f4(n, m):
+        return compute_hash(str(n) + str(m))
 
     def f(n, m):
         a = f1(n, m)
         b = f2(n, m)
         c = f3(n, m)
-        return a + len(b) + c
+        d = f4(n, m)
+        enable_siphash24()
+        return a + len(b) + c + d
 
     rtyper = support.annotate(f, [7, 9])
     jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
@@ -309,7 +316,8 @@
     for index, expected in [
             (0, EffectInfo.EF_ELIDABLE_CANNOT_RAISE),
             (1, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR),
-            (2, EffectInfo.EF_ELIDABLE_CAN_RAISE)]:
+            (2, EffectInfo.EF_ELIDABLE_CAN_RAISE),
+            (3, EffectInfo.EF_ELIDABLE_OR_MEMORYERROR)]:
         call_op = f_graph.startblock.operations[index]
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -26,7 +26,6 @@
 
 class AsmGcRootFrameworkGCTransformer(BaseFrameworkGCTransformer):
     _asmgcc_save_restore_arguments = None
-    _seen_gctransformer_hint_close_stack = None
 
     def push_roots(self, hop, keep_current_args=False):
         livevars = self.get_livevars_for_roots(hop, keep_current_args)
@@ -50,29 +49,28 @@
         hop.genop("direct_call", [c_asm_nocollect, name])
 
     def gct_direct_call(self, hop):
+        # just a sanity check: if we find a fnptr with the hint on the
+        # _callable, then we'd also find the hint by looking only at the
+        # graph.  We'll actually change this graph only later, in
+        # start_transforming_graph().
         fnptr = hop.spaceop.args[0].value
         try:
             close_stack = fnptr._obj._callable._gctransformer_hint_close_stack_
         except AttributeError:
+            pass
+        else:
+            assert fnptr._obj.graph.func is fnptr._obj._callable
+        BaseFrameworkGCTransformer.gct_direct_call(self, hop)
+
+    def start_transforming_graph(self, graph):
+        try:
+            close_stack = graph.func._gctransformer_hint_close_stack_
+        except AttributeError:
             close_stack = False
         if close_stack:
-            self.handle_call_with_close_stack(hop)
-        else:
-            BaseFrameworkGCTransformer.gct_direct_call(self, hop)
+            self._transform_hint_close_stack(graph)
 
-    def handle_call_with_close_stack(self, hop):
-        fnptr = hop.spaceop.args[0].value
-        if self._seen_gctransformer_hint_close_stack is None:
-            self._seen_gctransformer_hint_close_stack = {}
-        if fnptr._obj.graph not in self._seen_gctransformer_hint_close_stack:
-            self._transform_hint_close_stack(fnptr)
-            self._seen_gctransformer_hint_close_stack[fnptr._obj.graph] = True
-        #
-        livevars = self.push_roots(hop)
-        self.default(hop)
-        self.pop_roots(hop, livevars)
-
-    def _transform_hint_close_stack(self, fnptr):
+    def _transform_hint_close_stack(self, graph):
         # We cannot easily pass variable amount of arguments of the call
         # across the call to the pypy_asm_stackwalk helper.  So we store
         # them away and restore them.  More precisely, we need to
@@ -83,8 +81,8 @@
         sradict = self._asmgcc_save_restore_arguments
         sra = []     # list of pointers to raw-malloced containers for args
         seen = {}
-        FUNC1 = lltype.typeOf(fnptr).TO
-        for TYPE in FUNC1.ARGS:
+        ARGS = [v.concretetype for v in graph.getargs()]
+        for TYPE in ARGS:
             if isinstance(TYPE, lltype.Ptr):
                 TYPE = llmemory.Address
             num = seen.get(TYPE, 0)
@@ -98,8 +96,10 @@
             sra.append(sradict[key])
         #
         # make a copy of the graph that will reload the values
-        graph = fnptr._obj.graph
         graph2 = copygraph(graph)
+        del graph2.func   # otherwise, start_transforming_graph() will
+                          # again transform graph2, and we get an
+                          # infinite loop
         #
         # edit the original graph to only store the value of the arguments
         block = Block(graph.startblock.inputargs)
@@ -116,17 +116,18 @@
                 SpaceOperation("bare_setfield", [c_p, c_item0, v_arg], v_void))
         #
         # call asm_stackwalk(graph2)
-        FUNC2 = lltype.FuncType([], FUNC1.RESULT)
+        RESULT = graph.getreturnvar().concretetype
+        FUNC2 = lltype.FuncType([], RESULT)
         fnptr2 = lltype.functionptr(FUNC2,
-                                    fnptr._obj._name + '_reload',
+                                    graph.name + '_reload',
                                     graph=graph2)
         c_fnptr2 = Constant(fnptr2, lltype.Ptr(FUNC2))
         HELPERFUNC = lltype.FuncType([lltype.Ptr(FUNC2),
-                                      ASM_FRAMEDATA_HEAD_PTR], FUNC1.RESULT)
+                                      ASM_FRAMEDATA_HEAD_PTR], RESULT)
         v_asm_stackwalk = varoftype(lltype.Ptr(HELPERFUNC), "asm_stackwalk")
         block.operations.append(
             SpaceOperation("cast_pointer", [c_asm_stackwalk], v_asm_stackwalk))
-        v_result = varoftype(FUNC1.RESULT)
+        v_result = varoftype(RESULT)
         block.operations.append(
             SpaceOperation("indirect_call", [v_asm_stackwalk, c_fnptr2,
                                              c_gcrootanchor,
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -201,6 +201,9 @@
         self.var_last_needed_in = None
         self.curr_block = None
 
+    def start_transforming_graph(self, graph):
+        pass    # for asmgcc.py
+
     def transform_graph(self, graph):
         if graph in self.minimal_transform:
             if self.minimalgctransformer:
@@ -210,6 +213,7 @@
         if graph in self.seen_graphs:
             return
         self.seen_graphs.add(graph)
+        self.start_transforming_graph(graph)
 
         self.links_to_split = {} # link -> vars to pop_alive across the link
 
diff --git a/rpython/rlib/test/test_rsignal.py b/rpython/rlib/test/test_rsignal.py
--- a/rpython/rlib/test/test_rsignal.py
+++ b/rpython/rlib/test/test_rsignal.py
@@ -1,4 +1,4 @@
-import os, py
+import os, py, errno
 from rpython.translator.c.test.test_genc import compile
 from rpython.rlib import rsignal
 
@@ -37,3 +37,24 @@
 def test_compile():
     fn = compile(test_simple, [])
     fn()
+
+def test_compile_wakeup_fd():
+    def fn():
+        rd, wr = os.pipe()
+        rsignal.pypysig_set_wakeup_fd(wr, False)
+        for i in range(3):
+            rsignal.pypysig_setflag(rsignal.SIGUSR1)
+            os.kill(os.getpid(), rsignal.SIGUSR1)
+            check(rsignal.SIGUSR1)
+            check(-1)
+            check(-1)
+        x = os.read(rd, 10)
+        assert x == chr(rsignal.SIGUSR1) * 3
+        #
+        rsignal.pypysig_set_wakeup_fd(rd, False)   # can't write there
+        os.kill(os.getpid(), rsignal.SIGUSR1)
+
+    fn = compile(fn, [], return_stderr=True)
+    stderr = fn()
+    assert stderr.endswith('Exception ignored when trying to write to the '
+                           'signal wakeup fd: Errno %d\n' % errno.EBADF)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -539,7 +539,6 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
-    'call_at_startup':      LLOp(canrun=True),
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -742,9 +742,6 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
-def op_call_at_startup(init_func):
-    pass    # do nothing
-
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -60,7 +60,6 @@
         self.completed = False
 
         self.instrument_ncounter = 0
-        self.call_at_startup = set()
 
     def gettypedefnode(self, T, varlength=None):
         if varlength is None:
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -941,18 +941,3 @@
                 cdecl(typename, ''),
                 self.expr(op.args[0]),
                 self.expr(op.result))
-
-    def OP_CALL_AT_STARTUP(self, op):
-        c = op.args[0]
-        if not isinstance(c, Constant):
-            # Bah, maybe it comes from a same_as(const) just before...
-            # Can occur if running without backendopts
-            for op1 in self._current_block.operations:
-                if op1.result is op.args[0]:
-                    assert op1.opname == "same_as"
-                    c = op1.args[0]
-                    break
-            assert isinstance(c, Constant)
-        func = self.expr(c)
-        self.db.call_at_startup.add(func)
-        return '/* call_at_startup %s */' % (func,)
diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c
--- a/rpython/translator/c/src/signals.c
+++ b/rpython/translator/c/src/signals.c
@@ -2,6 +2,7 @@
 
 #include <limits.h>
 #include <stdlib.h>
+#include <errno.h>
 #ifdef _WIN32
 #include <process.h>
 #include <io.h>
@@ -82,27 +83,51 @@
       }
 }
 
+static void write_str(int fd, const char *p)
+{
+    int i = 0;
+    while (p[i] != '\x00')
+        i++;
+    (void)write(fd, p, i);
+}
+
 static void signal_setflag_handler(int signum)
 {
     pypysig_pushback(signum);
 
-    if (wakeup_fd != -1) 
-      {
+    /* Warning, this logic needs to be async-signal-safe */
+    if (wakeup_fd != -1) {
 #ifndef _WIN32
         ssize_t res;
 #else
         int res;
 #endif
-	if (wakeup_with_nul_byte)
-	{
-	    res = write(wakeup_fd, "\0", 1);
-	} else {
-	    unsigned char byte = (unsigned char)signum;
-	    res = write(wakeup_fd, &byte, 1);
-	}
-
-        /* the return value is ignored here */
-      }
+        int old_errno = errno;
+     retry:
+        if (wakeup_with_nul_byte) {
+            res = write(wakeup_fd, "\0", 1);
+        } else {
+            unsigned char byte = (unsigned char)signum;
+            res = write(wakeup_fd, &byte, 1);
+        }
+        if (res < 0) {
+            unsigned int e = (unsigned int)errno;
+            char c[27], *p;
+            if (e == EINTR)
+                goto retry;
+            write_str(2, "Exception ignored when trying to write to the "
+                         "signal wakeup fd: Errno ");
+            p = c + sizeof(c);
+            *--p = 0;
+            *--p = '\n';
+            do {
+                *--p = '0' + e % 10;
+                e /= 10;
+            } while (e != 0);
+            write_str(2, p);
+        }
+        errno = old_errno;
+    }
 }
 
 void pypysig_setflag(int signum)
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1063,23 +1063,39 @@
         assert out.strip() == expected
 
     def test_call_at_startup(self):
-        from rpython.rtyper.lltypesystem import lltype
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rtyper.annlowlevel import llhelper
+        from rpython.rtyper.extregistry import ExtRegistryEntry
+
         class State:
             seen = 0
         state = State()
         def startup():
             state.seen += 1
-        F = lltype.Ptr(lltype.FuncType([], lltype.Void))
+        def enablestartup():
+            "NOT_RPYTHON"
         def entry_point(argv):
             state.seen += 100
             assert state.seen == 101
             print 'ok'
-            ll = llhelper(F, startup)
-            llop.call_at_startup(lltype.Void, ll)
+            enablestartup()
             return 0
 
+        class Entry(ExtRegistryEntry):
+            _about_ = enablestartup
+
+            def compute_result_annotation(self):
+                bk = self.bookkeeper
+                s_callable = bk.immutablevalue(startup)
+                key = (enablestartup,)
+                bk.emulate_pbc_call(key, s_callable, [])
+
+            def specialize_call(self, hop):
+                hop.exception_cannot_occur()
+                bk = hop.rtyper.annotator.bookkeeper
+                s_callable = bk.immutablevalue(startup)
+                r_callable = hop.rtyper.getrepr(s_callable)
+                ll_init = r_callable.get_unique_llfn().value
+                bk.annotator.translator._call_at_startup.append(ll_init)
+
         t, cbuilder = self.compile(entry_point)
         out = cbuilder.cmdexec('')
         assert out.strip() == 'ok'

From pypy.commits at gmail.com  Thu Feb  2 11:26:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 02 Feb 2017 08:26:04 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <58935d9c.52a5df0a.6e805.0b7f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89903:cc6165679b16
Date: 2017-02-02 17:25 +0100
http://bitbucket.org/pypy/pypy/changeset/cc6165679b16/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -273,7 +273,7 @@
             i21 = getfield_gc_i(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
-            i24 = getfield_gc_i(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
+            i24 = getfield_gc_i(p17, descr=<FieldU .*W_ArrayBase.inst__buffer .*>)
             i25 = getarrayitem_raw_i(i24, 0, descr=<.*>)
             i27 = int_lt(1, i21)
             guard_false(i27, descr=...)

From pypy.commits at gmail.com  Thu Feb  2 16:38:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 02 Feb 2017 13:38:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Mess with import locks,
 when giving a .pyc file name on the command-line
Message-ID: <5893a6d8.9d711c0a.fd853.d10e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89904:85b6fb286e66
Date: 2017-02-02 18:19 +0100
http://bitbucket.org/pypy/pypy/changeset/85b6fb286e66/

Log:	Mess with import locks, when giving a .pyc file name on the command-
	line

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -728,8 +728,26 @@
             if IS_WINDOWS:
                 filename = filename.lower()
             if filename.endswith('.pyc') or filename.endswith('.pyo'):
+                # We don't actually load via SourcelessFileLoader
+                # because '__main__' must not be listed inside
+                # 'importlib._bootstrap._module_locks' (it deadlocks
+                # test_multiprocessing_main_handling.test_script_compiled)
+                from importlib._bootstrap_external import MAGIC_NUMBER
+                import marshal
                 loader = SourcelessFileLoader('__main__', filename)
-                args = (loader.load_module, loader.name)
+                mainmodule.__loader__ = loader
+                @hidden_applevel
+                def execfile(filename, namespace):
+                    with open(filename, 'rb') as f:
+                        if f.read(4) != MAGIC_NUMBER:
+                            raise RuntimeError("Bad magic number in .pyc file")
+                        if len(f.read(8)) != 8:
+                            raise RuntimeError("Truncated .pyc file")
+                        co = marshal.load(f)
+                    if type(co) is not type((lambda:0).__code__):
+                        raise RuntimeError("Bad code object in .pyc file")
+                    exec_(co, namespace)
+                args = (execfile, filename, mainmodule.__dict__)
             else:
                 filename = sys.argv[0]
                 for hook in sys.path_hooks:

From pypy.commits at gmail.com  Fri Feb  3 04:04:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 01:04:37 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: this was done already
Message-ID: <589447a5.958c1c0a.a270a.9d52@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5771:8e0025f7a61c
Date: 2017-02-03 10:04 +0100
http://bitbucket.org/pypy/extradoc/changeset/8e0025f7a61c/

Log:	this was done already

diff --git a/planning/py3.5/milestone-2-progress.rst b/planning/py3.5/milestone-2-progress.rst
--- a/planning/py3.5/milestone-2-progress.rst
+++ b/planning/py3.5/milestone-2-progress.rst
@@ -35,7 +35,7 @@
 * 'import stackless' fails
 
 * "except pyopcode.Return:" in pyframe can't be there, because that's
-  outside the JIT and it gives terrible performance
+  outside the JIT and it gives terrible performance (DONE)
   
 * PEP 475: Retry system calls failing with EINTR (DONE)
 

From pypy.commits at gmail.com  Fri Feb  3 05:09:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 02:09:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Re-enable accidentally disabled test.
 Fix it.
Message-ID: <589456d7.0a8edf0a.8962.45d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89905:5d3e217e3e64
Date: 2017-02-03 11:08 +0100
http://bitbucket.org/pypy/pypy/changeset/5d3e217e3e64/

Log:	Re-enable accidentally disabled test. Fix it.

diff --git a/rpython/rlib/_rweakvaldict.py b/rpython/rlib/_rweakvaldict.py
--- a/rpython/rlib/_rweakvaldict.py
+++ b/rpython/rlib/_rweakvaldict.py
@@ -119,7 +119,7 @@
     @jit.dont_look_inside
     def ll_get(self, d, llkey):
         if d.resize_counter < 0:
-            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
+            self.ll_weakdict_rehash_after_translation(d)
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         #llop.debug_print(lltype.Void, i, 'get')
@@ -139,7 +139,7 @@
     @jit.dont_look_inside
     def ll_set_nonnull(self, d, llkey, llvalue):
         if d.resize_counter < 0:
-            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
+            self.ll_weakdict_rehash_after_translation(d)
         hash = self.ll_keyhash(llkey)
         valueref = weakref_create(llvalue)    # GC effects here, before the rest
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
@@ -156,7 +156,7 @@
     @jit.dont_look_inside
     def ll_set_null(self, d, llkey):
         if d.resize_counter < 0:
-            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
+            self.ll_weakdict_rehash_after_translation(d)
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         if d.entries.everused(i):
@@ -180,6 +180,15 @@
         d.num_items = num_items
         rdict.ll_dict_resize(d)
 
+    def ll_weakdict_rehash_after_translation(self, d):
+        # recompute all hashes.  See comment in rordereddict.py,
+        # ll_dict_rehash_after_translation().
+        entries = d.entries
+        for i in range(len(entries)):
+            self.ll_keyhash(entries[i].key)
+        self.ll_weakdict_resize(d)
+        assert d.resize_counter >= 0
+
 def specialize_make_weakdict(hop):
     hop.exception_cannot_occur()
     v_d = hop.gendirectcall(hop.r_result.ll_new_weakdict)
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -230,7 +230,7 @@
     fc = compile(f, [], gcpolicy="boehm", rweakref=True)
     fc()
 
-def _test_translation_prebuilt_2():
+def test_translation_prebuilt_2():
     from rpython.rlib import rsiphash
     d = RWeakValueDictionary(str, X)
     k1 = "key1"; k2 = "key2"

From pypy.commits at gmail.com  Fri Feb  3 05:09:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 02:09:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <589456f1.c80e1c0a.43bdb.b580@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89906:b96d7d91a143
Date: 2017-02-03 11:09 +0100
http://bitbucket.org/pypy/pypy/changeset/b96d7d91a143/

Log:	hg merge default

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -235,7 +235,7 @@
             i21 = getfield_gc_i(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
-            i24 = getfield_gc_i(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
+            i24 = getfield_gc_i(p17, descr=<FieldU .*W_ArrayBase.inst__buffer .*>)
             i25 = getarrayitem_raw_i(i24, 0, descr=<.*>)
             i27 = int_lt(1, i21)
             guard_false(i27, descr=...)
diff --git a/rpython/rlib/_rweakvaldict.py b/rpython/rlib/_rweakvaldict.py
--- a/rpython/rlib/_rweakvaldict.py
+++ b/rpython/rlib/_rweakvaldict.py
@@ -119,7 +119,7 @@
     @jit.dont_look_inside
     def ll_get(self, d, llkey):
         if d.resize_counter < 0:
-            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
+            self.ll_weakdict_rehash_after_translation(d)
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         #llop.debug_print(lltype.Void, i, 'get')
@@ -139,7 +139,7 @@
     @jit.dont_look_inside
     def ll_set_nonnull(self, d, llkey, llvalue):
         if d.resize_counter < 0:
-            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
+            self.ll_weakdict_rehash_after_translation(d)
         hash = self.ll_keyhash(llkey)
         valueref = weakref_create(llvalue)    # GC effects here, before the rest
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
@@ -156,7 +156,7 @@
     @jit.dont_look_inside
     def ll_set_null(self, d, llkey):
         if d.resize_counter < 0:
-            self.ll_weakdict_resize(d)  # initialize prebuilt dicts at runtime
+            self.ll_weakdict_rehash_after_translation(d)
         hash = self.ll_keyhash(llkey)
         i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK
         if d.entries.everused(i):
@@ -180,6 +180,15 @@
         d.num_items = num_items
         rdict.ll_dict_resize(d)
 
+    def ll_weakdict_rehash_after_translation(self, d):
+        # recompute all hashes.  See comment in rordereddict.py,
+        # ll_dict_rehash_after_translation().
+        entries = d.entries
+        for i in range(len(entries)):
+            self.ll_keyhash(entries[i].key)
+        self.ll_weakdict_resize(d)
+        assert d.resize_counter >= 0
+
 def specialize_make_weakdict(hop):
     hop.exception_cannot_occur()
     v_d = hop.gendirectcall(hop.r_result.ll_new_weakdict)
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -446,8 +446,85 @@
             time.sleep(1)  # give the GDB time to attach
 
 else:
+    def make_vs_attach_eci():
+        # The COM interface to the Debugger has to be compiled as a .cpp file by
+        # Visual C. So we generate the source and then add a commandline switch
+        # to treat this source file as C++
+        import os
+        eci = ExternalCompilationInfo(post_include_bits=["""
+#ifdef __cplusplus
+extern "C" {
+#endif
+RPY_EXPORTED void AttachToVS();
+#ifdef __cplusplus
+}
+#endif
+                                      """],
+                                      separate_module_sources=["""
+#import "libid:80cc9f66-e7d8-4ddd-85b6-d9e6cd0e93e2" version("8.0") lcid("0") raw_interfaces_only named_guids
+extern "C" RPY_EXPORTED void AttachToVS() {
+    CoInitialize(0);
+    HRESULT hr;
+    CLSID Clsid;
+
+    CLSIDFromProgID(L"VisualStudio.DTE", &Clsid);
+    IUnknown *Unknown;
+    if (FAILED(GetActiveObject(Clsid, 0, &Unknown))) {
+        puts("Could not attach to Visual Studio (is it not running?");
+        return;
+    }
+
+    EnvDTE::_DTE *Interface;
+    hr = Unknown->QueryInterface(&Interface);
+    if (FAILED(GetActiveObject(Clsid, 0, &Unknown))) {
+        puts("Could not open COM interface to Visual Studio (no permissions?)");
+        return;
+    }
+
+    EnvDTE::Debugger *Debugger;
+    puts("Waiting for Visual Studio Debugger to become idle");
+    while (FAILED(Interface->get_Debugger(&Debugger)));
+
+    EnvDTE::Processes *Processes;
+    while (FAILED(Debugger->get_LocalProcesses(&Processes)));
+
+    long Count = 0;
+    if (FAILED(Processes->get_Count(&Count))) {
+        puts("Cannot query Process count");
+    }
+
+    for (int i = 0; i <= Count; i++) {
+        EnvDTE::Process *Process;
+        if (FAILED(Processes->Item(variant_t(i), &Process))) {
+            continue;
+        }
+
+        long ProcessID;
+        while (FAILED(Process->get_ProcessID(&ProcessID)));
+
+        if (ProcessID == GetProcessId(GetCurrentProcess())) {
+            printf("Found process ID %d\\n", ProcessID);
+            Process->Attach();
+            Debugger->Break(false);
+            CoUninitialize();
+            return;
+        }
+    }
+}
+                                      """]
+        )
+        eci = eci.convert_sources_to_files()
+        d = eci._copy_attributes()
+        cfile = d['separate_module_files'][0]
+        cppfile = cfile.replace(".c", "_vsdebug.cpp")
+        os.rename(cfile, cppfile)
+        d['separate_module_files'] = [cppfile]
+        return ExternalCompilationInfo(**d)
+
+    ll_attach = rffi.llexternal("AttachToVS", [], lltype.Void,
+                                compilation_info=make_vs_attach_eci())
     def impl_attach_gdb():
-        print "Don't know how to attach GDB on Windows"
+        ll_attach()
 
 register_external(attach_gdb, [], result=None,
                   export_name="impl_attach_gdb", llimpl=impl_attach_gdb)
diff --git a/rpython/rlib/test/test_rweakvaldict.py b/rpython/rlib/test/test_rweakvaldict.py
--- a/rpython/rlib/test/test_rweakvaldict.py
+++ b/rpython/rlib/test/test_rweakvaldict.py
@@ -230,7 +230,7 @@
     fc = compile(f, [], gcpolicy="boehm", rweakref=True)
     fc()
 
-def _test_translation_prebuilt_2():
+def test_translation_prebuilt_2():
     from rpython.rlib import rsiphash
     d = RWeakValueDictionary(str, X)
     k1 = "key1"; k2 = "key2"
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -379,9 +379,9 @@
             no_precompile = []
             for f in list(no_precompile_cfiles):
                 f = m.pathrel(py.path.local(f))
-                if f not in no_precompile and f.endswith('.c'):
+                if f not in no_precompile and (f.endswith('.c') or f.endswith('.cpp')):
                     no_precompile.append(f)
-                    target = f[:-1] + 'obj'
+                    target = f[:f.rfind('.')] + '.obj'
                     rules.append((target, f,
                         '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
                         '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f)))

From pypy.commits at gmail.com  Fri Feb  3 05:17:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 02:17:10 -0800 (PST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <589458a6.2687df0a.bedd8.624f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89907:ad26698c18d7
Date: 2017-02-03 11:16 +0100
http://bitbucket.org/pypy/pypy/changeset/ad26698c18d7/

Log:	fix

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -65,6 +65,8 @@
     w_KeyError = W_TypeObject("KeyError")
     w_SystemExit = W_TypeObject("SystemExit")
     w_KeyboardInterrupt = W_TypeObject("KeyboardInterrupt")
+    w_RuntimeError = W_TypeObject("RuntimeError")
+    w_RecursionError = W_TypeObject("RecursionError")   # py3.5
     w_VisibleDeprecationWarning = W_TypeObject("VisibleDeprecationWarning")
     w_None = W_Root()
 

From pypy.commits at gmail.com  Fri Feb  3 06:43:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 03:43:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test and fix
Message-ID: <58946cee.c80e1c0a.43bdb.dd90@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89908:885116967761
Date: 2017-02-03 12:43 +0100
http://bitbucket.org/pypy/pypy/changeset/885116967761/

Log:	Test and fix

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -522,8 +522,18 @@
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue, w_name):
         name = space.str_or_None_w(w_name)
+        #
+        if sys.platform != 'win32' and name is not None:
+            # like CPython, in this case ignore 'w_handle'
+            try:
+                handle = create_semaphore(space, name, 0, maxvalue)
+            except OSError as e:
+                raise wrap_oserror(space, e)
+        else:
+            handle = handle_w(space, w_handle)
+        #
         self = space.allocate_instance(W_SemLock, w_cls)
-        self.__init__(space, handle_w(space, w_handle), kind, maxvalue, name)
+        self.__init__(space, handle, kind, maxvalue, name)
         return space.wrap(self)
 
     def enter(self, space):
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -108,7 +108,9 @@
         maxvalue = 1
         sem = SemLock(kind, value, maxvalue, "4", unlink=True)
 
-        sem2 = SemLock._rebuild(sem.handle, kind, value, "10")
+        sem2 = SemLock._rebuild(sem.handle, kind, value, "92")
+        assert sem.handle != sem2.handle
+        sem2 = SemLock._rebuild(sem.handle, kind, value, None)
         assert sem.handle == sem2.handle
 
     def test_semaphore_contextmanager(self):

From pypy.commits at gmail.com  Fri Feb  3 06:53:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 03:53:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Kill _multiprocessing.Connection,
 like it seems to have occurred in
Message-ID: <58946f55.0a081c0a.d6e12.e1d2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89909:170cf64817b0
Date: 2017-02-03 12:53 +0100
http://bitbucket.org/pypy/pypy/changeset/170cf64817b0/

Log:	Kill _multiprocessing.Connection, like it seems to have occurred in
	CPython. (Just in case, hg history can let us recover it.)

diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py
--- a/pypy/module/_multiprocessing/__init__.py
+++ b/pypy/module/_multiprocessing/__init__.py
@@ -5,7 +5,6 @@
 class Module(MixedModule):
 
     interpleveldefs = {
-        'Connection'      : 'interp_connection.W_FileConnection',
         'SemLock'         : 'interp_semaphore.W_SemLock',
         'sem_unlink'      : 'interp_semaphore.semaphore_unlink',
         'address_of_buffer' : 'interp_memory.address_of_buffer',
@@ -15,12 +14,5 @@
     }
 
     if sys.platform == 'win32':
-        interpleveldefs['PipeConnection'] = \
-            'interp_connection.W_PipeConnection'
         interpleveldefs['win32'] = 'interp_win32.win32_namespace(space)'
         del interpleveldefs['sem_unlink']
-
-    def init(self, space):
-        MixedModule.init(self, space)
-        from pypy.module._multiprocessing.interp_connection import State
-        space.fromcache(State).init(space)
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
deleted file mode 100644
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ /dev/null
@@ -1,547 +0,0 @@
-import sys
-from errno import EINTR
-
-from rpython.rlib import rpoll, rsocket
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rarithmetic import intmask
-from rpython.rtyper.lltypesystem import lltype, rffi
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
-from pypy.interpreter.gateway import (
-    WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
-from pypy.interpreter.typedef import GetSetProperty, TypeDef
-
-READABLE, WRITABLE = range(1, 3)
-PY_SSIZE_T_MAX = sys.maxint
-PY_SSIZE_T_MIN = -sys.maxint - 1
-
-class State(object):
-    def __init__(self, space):
-        pass
-
-    def init(self, space):
-        w_builtins = space.getbuiltinmodule('builtins')
-        w_module = space.call_method(
-            w_builtins, '__import__', space.wrap("multiprocessing"))
-        self.w_BufferTooShort = space.getattr(w_module, space.wrap("BufferTooShort"))
-
-        self.w_picklemodule = space.call_method(
-            w_builtins, '__import__', space.wrap("pickle"))
-
-def BufferTooShort(space, w_data):
-    state = space.fromcache(State)
-    if not we_are_translated() and not hasattr(state, 'w_BufferTooShort'):
-        state.init(space)   # xxx for test/test_connection.py
-    w_BufferTooShort = state.w_BufferTooShort
-    return OperationError(w_BufferTooShort, w_data)
-
-def w_handle(space, handle):
-    return space.wrap(rffi.cast(rffi.INTPTR_T, handle))
-
-
-class W_BaseConnection(W_Root):
-    BUFFER_SIZE = 1024
-    buffer = lltype.nullptr(rffi.CCHARP.TO)
-
-    def __init__(self, space, flags):
-        self.flags = flags
-        self.buffer = lltype.malloc(rffi.CCHARP.TO, self.BUFFER_SIZE,
-                                    flavor='raw')
-        self.register_finalizer(space)
-
-    def _finalize_(self):
-        buf = self.buffer
-        if buf:
-            self.buffer = lltype.nullptr(rffi.CCHARP.TO)
-            lltype.free(buf, flavor='raw')
-        try:
-            self.do_close()
-        except OSError:
-            pass
-
-    # Abstract methods
-    def do_close(self):
-        raise NotImplementedError
-    def is_valid(self):
-        return False
-    def do_send_string(self, space, buf, offset, size):
-        raise NotImplementedError
-    def do_recv_string(self, space, buflength, maxlength):
-        raise NotImplementedError
-    def do_poll(self, space, timeout):
-        raise NotImplementedError
-
-    def close(self):
-        self.do_close()
-
-    def closed_get(self, space):
-        return space.newbool(not self.is_valid())
-    def readable_get(self, space):
-        return space.newbool(bool(self.flags & READABLE))
-    def writable_get(self, space):
-        return space.newbool(bool(self.flags & WRITABLE))
-
-    def _repr(self, space, handle):
-        index = self.flags - 1
-        conn_type = [u"read-only", u"write-only", u"read-write"][index]
-        return space.wrap(u"<%s %s, handle %d>" % (
-                conn_type, space.type(self).getname(space), handle))
-
-    def descr_repr(self, space):
-        raise NotImplementedError
-
-    def _check_readable(self, space):
-        if not self.flags & READABLE:
-            raise oefmt(space.w_IOError, "connection is write-only")
-    def _check_writable(self, space):
-        if not self.flags & WRITABLE:
-            raise oefmt(space.w_IOError, "connection is read-only")
-
-    @unwrap_spec(offset='index', size='index')
-    def send_bytes(self, space, w_buf, offset=0, size=PY_SSIZE_T_MIN):
-        buf = space.getarg_w('s*', w_buf).as_str()
-        length = len(buf)
-        self._check_writable(space)
-        if offset < 0:
-            raise oefmt(space.w_ValueError, "offset is negative")
-        if length < offset:
-            raise oefmt(space.w_ValueError, "buffer length < offset")
-
-        if size == PY_SSIZE_T_MIN:
-            size = length - offset
-        elif size < 0:
-            raise oefmt(space.w_ValueError, "size is negative")
-        elif offset + size > length:
-            raise oefmt(space.w_ValueError, "buffer length > offset + size")
-
-        self.do_send_string(space, buf, offset, size)
-
-    @unwrap_spec(maxlength='index')
-    def recv_bytes(self, space, maxlength=PY_SSIZE_T_MAX):
-        self._check_readable(space)
-        if maxlength < 0:
-            raise oefmt(space.w_ValueError, "maxlength < 0")
-
-        res, newbuf = self.do_recv_string(
-            space, self.BUFFER_SIZE, maxlength)
-        try:
-            if newbuf:
-                return space.newbytes(rffi.charpsize2str(newbuf, res))
-            else:
-                return space.newbytes(rffi.charpsize2str(self.buffer, res))
-        finally:
-            if newbuf:
-                rffi.free_charp(newbuf)
-
-    @unwrap_spec(offset='index')
-    def recv_bytes_into(self, space, w_buffer, offset=0):
-        rwbuffer = space.writebuf_w(w_buffer)
-        length = rwbuffer.getlength()
-
-        res, newbuf = self.do_recv_string(
-            space, length - offset, PY_SSIZE_T_MAX)
-        try:
-            if newbuf:
-                raise BufferTooShort(space, space.newbytes(
-                    rffi.charpsize2str(newbuf, res)))
-            rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res))
-        finally:
-            if newbuf:
-                rffi.free_charp(newbuf)
-
-        return space.wrap(res)
-
-    def send(self, space, w_obj):
-        self._check_writable(space)
-
-        w_picklemodule = space.fromcache(State).w_picklemodule
-        w_protocol = space.getattr(
-            w_picklemodule, space.wrap("HIGHEST_PROTOCOL"))
-        w_pickled = space.call_method(
-            w_picklemodule, "dumps", w_obj, w_protocol)
-
-        buf = space.str_w(w_pickled)
-        self.do_send_string(space, buf, 0, len(buf))
-
-    def recv(self, space):
-        self._check_readable(space)
-
-        res, newbuf = self.do_recv_string(
-            space, self.BUFFER_SIZE, PY_SSIZE_T_MAX)
-        try:
-            if newbuf:
-                w_received = space.newbytes(rffi.charpsize2str(newbuf, res))
-            else:
-                w_received = space.newbytes(rffi.charpsize2str(self.buffer, res))
-        finally:
-            if newbuf:
-                rffi.free_charp(newbuf)
-
-        w_builtins = space.getbuiltinmodule('builtins')
-        w_picklemodule = space.fromcache(State).w_picklemodule
-        w_unpickled = space.call_method(
-            w_picklemodule, "loads", w_received)
-
-        return w_unpickled
-
-    @unwrap_spec(w_timeout=WrappedDefault(0.0))
-    def poll(self, space, w_timeout):
-        self._check_readable(space)
-        if space.is_w(w_timeout, space.w_None):
-            timeout = -1.0 # block forever
-        else:
-            timeout = space.float_w(w_timeout)
-            if timeout < 0.0:
-                timeout = 0.0
-        return space.newbool(self.do_poll(space, timeout))
-
-W_BaseConnection.typedef = TypeDef(
-    'BaseConnection',
-    __repr__ = interpindirect2app(W_BaseConnection.descr_repr),
-    closed = GetSetProperty(W_BaseConnection.closed_get),
-    readable = GetSetProperty(W_BaseConnection.readable_get),
-    writable = GetSetProperty(W_BaseConnection.writable_get),
-
-    send_bytes = interp2app(W_BaseConnection.send_bytes),
-    recv_bytes = interp2app(W_BaseConnection.recv_bytes),
-    recv_bytes_into = interp2app(W_BaseConnection.recv_bytes_into),
-    send = interp2app(W_BaseConnection.send),
-    recv = interp2app(W_BaseConnection.recv),
-    poll = interp2app(W_BaseConnection.poll),
-    close = interp2app(W_BaseConnection.close),
-    )
-
-class W_FileConnection(W_BaseConnection):
-    INVALID_HANDLE_VALUE = -1
-    fd = INVALID_HANDLE_VALUE
-
-    if sys.platform == 'win32':
-        def WRITE(self, data):
-            from rpython.rlib._rsocket_rffi import send, geterrno
-            length = send(self.fd, data, len(data), 0)
-            if length < 0:
-                raise WindowsError(geterrno(), "send")
-            return length
-        def READ(self, size):
-            from rpython.rlib._rsocket_rffi import socketrecv, geterrno
-            with rffi.scoped_alloc_buffer(size) as buf:
-                length = socketrecv(self.fd, buf.raw, buf.size, 0)
-                if length < 0:
-                    raise WindowsError(geterrno(), "recv")
-                return buf.str(length)
-        def CLOSE(self):
-            from rpython.rlib._rsocket_rffi import socketclose
-            socketclose(self.fd)
-    else:
-        def WRITE(self, data):
-            import os
-            return os.write(self.fd, data)
-        def READ(self, length):
-            import os
-            return os.read(self.fd, length)
-        def CLOSE(self):
-            import os
-            try:
-                os.close(self.fd)
-            except OSError:
-                pass
-
-    def __init__(self, space, fd, flags):
-        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
-            raise oefmt(space.w_IOError, "invalid handle %d", fd)
-        W_BaseConnection.__init__(self, space, flags)
-        self.fd = fd
-
-    @unwrap_spec(fd=int, readable=bool, writable=bool)
-    def descr_new_file(space, w_subtype, fd, readable=True, writable=True):
-        flags = (readable and READABLE) | (writable and WRITABLE)
-
-        self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, space, fd, flags)
-        return space.wrap(self)
-
-    def descr_repr(self, space):
-        return self._repr(space, self.fd)
-
-    def fileno(self, space):
-        return space.wrap(self.fd)
-
-    def is_valid(self):
-        return self.fd != self.INVALID_HANDLE_VALUE
-
-    def do_close(self):
-        if self.is_valid():
-            self.CLOSE()
-            self.fd = self.INVALID_HANDLE_VALUE
-
-    def do_send_string(self, space, buf, offset, size):
-        # Since str2charp copies the buf anyway, always combine the
-        # "header" and the "body" of the message and send them at once.
-        message = lltype.malloc(rffi.CCHARP.TO, size + 4, flavor='raw')
-        try:
-            length = rffi.r_uint(rsocket.htonl(
-                    rffi.cast(lltype.Unsigned, size)))
-            rffi.cast(rffi.UINTP, message)[0] = length
-            i = size - 1
-            while i >= 0:
-                message[4 + i] = buf[offset + i]
-                i -= 1
-            self._sendall(space, message, size + 4)
-        finally:
-            lltype.free(message, flavor='raw')
-
-    def do_recv_string(self, space, buflength, maxlength):
-        with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1) as length_ptr:
-            self._recvall(space, rffi.cast(rffi.CCHARP, length_ptr), 4)
-            length = intmask(rsocket.ntohl(
-                    rffi.cast(lltype.Unsigned, length_ptr[0])))
-        if length > maxlength: # bad message, close connection
-            self.flags &= ~READABLE
-            if self.flags == 0:
-                self.close()
-            raise oefmt(space.w_IOError, "bad message length")
-
-        if length <= buflength:
-            self._recvall(space, self.buffer, length)
-            return length, lltype.nullptr(rffi.CCHARP.TO)
-        else:
-            newbuf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
-            self._recvall(space, newbuf, length)
-            return length, newbuf
-
-    def _sendall(self, space, message, size):
-        while size > 0:
-            # XXX inefficient
-            data = rffi.charpsize2str(message, size)
-            try:
-                count = self.WRITE(data)
-            except OSError as e:
-                if e.errno == EINTR:
-                    space.getexecutioncontext().checksignals()
-                    continue
-                raise wrap_oserror(space, e)
-            size -= count
-            message = rffi.ptradd(message, count)
-
-    def _recvall(self, space, buf, length):
-        length = intmask(length)
-        remaining = length
-        while remaining > 0:
-            try:
-                data = self.READ(remaining)
-            except OSError as e:
-                if e.errno == EINTR:
-                    space.getexecutioncontext().checksignals()
-                    continue
-                raise wrap_oserror(space, e)
-            count = len(data)
-            if count == 0:
-                if remaining == length:
-                    raise OperationError(space.w_EOFError, space.w_None)
-                else:
-                    raise oefmt(space.w_IOError,
-                                "got end of file during message")
-            # XXX inefficient
-            for i in range(count):
-                buf[i] = data[i]
-            remaining -= count
-            buf = rffi.ptradd(buf, count)
-
-    if sys.platform == 'win32':
-        def _check_fd(self):
-            return self.fd >= 0
-    else:
-        def _check_fd(self):
-            return self.fd >= 0 and self.fd < rpoll.FD_SETSIZE
-
-    def do_poll(self, space, timeout):
-        if not self._check_fd():
-            raise oefmt(space.w_IOError, "handle out of range in select()")
-        r, w, e = rpoll.select([self.fd], [], [], timeout, handle_eintr=True)
-        return bool(r)
-
-W_FileConnection.typedef = TypeDef(
-    '_multiprocessing.Connection', W_BaseConnection.typedef,
-    __new__ = interp2app(W_FileConnection.descr_new_file.im_func),
-    fileno = interp2app(W_FileConnection.fileno),
-)
-
-class W_PipeConnection(W_BaseConnection):
-    if sys.platform == 'win32':
-        from rpython.rlib.rwin32 import INVALID_HANDLE_VALUE
-
-    def __init__(self, space, handle, flags):
-        W_BaseConnection.__init__(self, space, flags)
-        self.handle = handle
-
-    @unwrap_spec(readable=bool, writable=bool)
-    def descr_new_pipe(space, w_subtype, w_handle, readable=True,
-                       writable=True):
-        from pypy.module._multiprocessing.interp_win32 import handle_w
-        handle = handle_w(space, w_handle)
-        flags = (readable and READABLE) | (writable and WRITABLE)
-
-        self = space.allocate_instance(W_PipeConnection, w_subtype)
-        W_PipeConnection.__init__(self, space, handle, flags)
-        return space.wrap(self)
-
-    def descr_repr(self, space):
-        return self._repr(space, rffi.cast(rffi.INTPTR_T, self.handle))
-
-    def is_valid(self):
-        return self.handle != self.INVALID_HANDLE_VALUE
-
-    def fileno(self, space):
-        return w_handle(space, self.handle)
-
-    def do_close(self):
-        from rpython.rlib.rwin32 import CloseHandle
-        if self.is_valid():
-            CloseHandle(self.handle)
-            self.handle = self.INVALID_HANDLE_VALUE
-
-    def do_send_string(self, space, buf, offset, size):
-        from pypy.module._multiprocessing.interp_win32 import (
-            _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
-        from rpython.rlib import rwin32
-
-        with rffi.scoped_view_charp(buf) as charp:
-            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                        flavor='raw')
-            try:
-                result = _WriteFile(
-                    self.handle, rffi.ptradd(charp, offset),
-                    size, written_ptr, rffi.NULL)
-
-                if (result == 0 and
-                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                    raise oefmt(space.w_ValueError,
-                                "Cannot send %d bytes over connection", size)
-            finally:
-                lltype.free(written_ptr, flavor='raw')
-
-    def do_recv_string(self, space, buflength, maxlength):
-        from pypy.module._multiprocessing.interp_win32 import (
-            _ReadFile, _PeekNamedPipe, ERROR_BROKEN_PIPE, ERROR_MORE_DATA)
-        from rpython.rlib import rwin32
-        from pypy.interpreter.error import wrap_windowserror
-
-        read_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                 flavor='raw')
-        left_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                 flavor='raw')
-        try:
-            result = _ReadFile(self.handle,
-                               self.buffer, min(self.BUFFER_SIZE, buflength),
-                               read_ptr, rffi.NULL)
-            if result:
-                return intmask(read_ptr[0]), lltype.nullptr(rffi.CCHARP.TO)
-
-            err = rwin32.GetLastError_saved()
-            if err == ERROR_BROKEN_PIPE:
-                raise OperationError(space.w_EOFError, space.w_None)
-            elif err != ERROR_MORE_DATA:
-                raise wrap_windowserror(space, WindowsError(err, "_ReadFile"))
-
-            # More data...
-            if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
-                                  lltype.nullptr(rwin32.LPDWORD.TO),
-                                  lltype.nullptr(rwin32.LPDWORD.TO),
-                                  left_ptr):
-                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-
-            length = intmask(read_ptr[0] + left_ptr[0])
-            if length > maxlength: # bad message, close connection
-                self.flags &= ~READABLE
-                if self.flags == 0:
-                    self.close()
-                raise oefmt(space.w_IOError, "bad message length")
-
-            newbuf = lltype.malloc(rffi.CCHARP.TO, length + 1, flavor='raw')
-            for i in range(read_ptr[0]):
-                newbuf[i] = self.buffer[i]
-
-            result = _ReadFile(self.handle,
-                               rffi.ptradd(newbuf, read_ptr[0]), left_ptr[0],
-                               read_ptr, rffi.NULL)
-            if not result:
-                rffi.free_charp(newbuf)
-                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-
-            assert read_ptr[0] == left_ptr[0]
-            return length, newbuf
-        finally:
-            lltype.free(read_ptr, flavor='raw')
-            lltype.free(left_ptr, flavor='raw')
-
-    def do_poll(self, space, timeout):
-        from pypy.module._multiprocessing.interp_win32 import (
-            _PeekNamedPipe, _GetTickCount, _Sleep)
-        from rpython.rlib import rwin32
-        from pypy.interpreter.error import wrap_windowserror
-        bytes_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                 flavor='raw')
-        try:
-            if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
-                                  lltype.nullptr(rwin32.LPDWORD.TO),
-                                  bytes_ptr,
-                                  lltype.nullptr(rwin32.LPDWORD.TO)):
-                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-            bytes = bytes_ptr[0]
-        finally:
-            lltype.free(bytes_ptr, flavor='raw')
-
-        if timeout == 0.0:
-            return bytes > 0
-
-        block = timeout < 0
-        if not block:
-            # XXX does not check for overflow
-            deadline = intmask(_GetTickCount()) + int(1000 * timeout + 0.5)
-        else:
-            deadline = 0
-
-        _Sleep(0)
-
-        delay = 1
-        while True:
-            bytes_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                     flavor='raw')
-            try:
-                if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
-                                      lltype.nullptr(rwin32.LPDWORD.TO),
-                                      bytes_ptr,
-                                      lltype.nullptr(rwin32.LPDWORD.TO)):
-                    raise wrap_windowserror(space,
-                                            rwin32.lastSavedWindowsError())
-                bytes = bytes_ptr[0]
-            finally:
-                lltype.free(bytes_ptr, flavor='raw')
-
-            if bytes > 0:
-                return True
-
-            if not block:
-                now = intmask(_GetTickCount())
-                if now > deadline:
-                    return False
-                diff = deadline - now
-                if delay > diff:
-                    delay = diff
-            else:
-                delay += 1
-
-            if delay >= 20:
-                delay = 20
-            _Sleep(delay)
-
-            # check for signals
-            # PyErr_CheckSignals()
-
-if sys.platform == 'win32':
-    W_PipeConnection.typedef = TypeDef(
-        '_multiprocessing.PipeConnection', W_BaseConnection.typedef,
-        __new__ = interp2app(W_PipeConnection.descr_new_pipe.im_func),
-        fileno = interp2app(W_PipeConnection.fileno),
-    )
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -13,7 +13,6 @@
 from pypy.interpreter.error import oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import GetSetProperty, TypeDef
-from pypy.module._multiprocessing.interp_connection import w_handle
 
 RECURSIVE_MUTEX, SEMAPHORE = range(2)
 
@@ -455,7 +454,8 @@
         return space.newint(self.maxvalue)
 
     def handle_get(self, space):
-        return w_handle(space, self.handle)
+        h = rffi.cast(rffi.INTPTR_T, self.handle)
+        return space.wrap(h)
 
     def get_count(self, space):
         return space.wrap(self.count)
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
deleted file mode 100644
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ /dev/null
@@ -1,200 +0,0 @@
-import py
-import sys
-from pypy.interpreter.gateway import interp2app, W_Root
-
-class TestImport:
-    def test_simple(self):
-        from pypy.module._multiprocessing import interp_connection
-        from pypy.module._multiprocessing import interp_semaphore
-
-class AppTestBufferTooShort:
-    spaceconfig = {'usemodules': ['_multiprocessing', 'thread', 'signal',
-                                  'select', 'struct', 'binascii',
-                                  '_posixsubprocess']}
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_rawffi')
-    else:
-        spaceconfig['usemodules'].append('fcntl')
-
-
-    def setup_class(cls):
-        if cls.runappdirect:
-            def raiseBufferTooShort(self, data):
-                import multiprocessing
-                raise multiprocessing.BufferTooShort(data)
-            cls.w_raiseBufferTooShort = raiseBufferTooShort
-        else:
-            from pypy.module._multiprocessing import interp_connection
-            def raiseBufferTooShort(space, w_data):
-                raise interp_connection.BufferTooShort(space, w_data)
-            cls.w_raiseBufferTooShort = cls.space.wrap(
-                interp2app(raiseBufferTooShort))
-
-    def test_exception(self):
-        import multiprocessing
-        try:
-            self.raiseBufferTooShort("data")
-        except multiprocessing.BufferTooShort as e:
-            assert isinstance(e, multiprocessing.ProcessError)
-            assert e.args == ("data",)
-
-class BaseConnectionTest(object):
-    def test_connection(self):
-        rhandle, whandle = self.make_pair()
-
-        whandle.send_bytes(b"abc")
-        assert rhandle.recv_bytes(100) == b"abc"
-
-        obj = [1, 2.0, "hello"]
-        whandle.send(obj)
-        obj2 = rhandle.recv()
-        assert obj == obj2
-
-    def test_poll(self):
-        rhandle, whandle = self.make_pair()
-
-        assert rhandle.poll() == False
-        assert rhandle.poll(1) == False
-        whandle.send(1)
-        import time; time.sleep(0.1)  # give it time to arrive :-)
-        assert rhandle.poll() == True
-        assert rhandle.poll(None) == True
-        assert rhandle.recv() == 1
-        assert rhandle.poll() == False
-        raises(IOError, whandle.poll)
-
-    def test_read_into(self):
-        import array, multiprocessing
-        rhandle, whandle = self.make_pair()
-
-        obj = [1, 2.0, "hello"]
-        whandle.send(obj)
-        buffer = array.array('b', [0]*10)
-        raises(multiprocessing.BufferTooShort, rhandle.recv_bytes_into, buffer)
-        assert rhandle.readable
-
-class AppTestWinpipeConnection(BaseConnectionTest):
-    spaceconfig = {
-        "usemodules": [
-            '_multiprocessing', 'thread', 'signal', 'struct', 'array',
-            'itertools', '_socket', 'binascii',
-        ]
-    }
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_rawffi')
-
-    def setup_class(cls):
-        if sys.platform != "win32":
-            py.test.skip("win32 only")
-
-        if not cls.runappdirect:
-            space = cls.space
-            # stubs for some modules,
-            # just for multiprocessing to import correctly on Windows
-            w_modules = space.sys.get('modules')
-            space.setitem(w_modules, space.wrap('msvcrt'), space.sys)
-        else:
-            import _multiprocessing
-
-    def w_make_pair(self):
-        import multiprocessing
-
-        return multiprocessing.Pipe(duplex=False)
-
-
-class AppTestSocketConnection(BaseConnectionTest):
-    spaceconfig = {
-        "usemodules": [
-            '_multiprocessing', 'thread', 'signal', 'struct', 'array',
-            '_socket', 'binascii', 'select', '_posixsubprocess']
-    }
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_rawffi')
-    else:
-        spaceconfig['usemodules'].append('fcntl')
-
-    def setup_class(cls):
-        cls.w_connections = cls.space.newlist([])
-
-    def w_socketpair(self):
-        "A socket.socketpair() that works on Windows"
-        import errno
-        import socket
-
-        serverSocket = socket.socket()
-        serverSocket.bind(('127.0.0.1', 0))
-        serverSocket.listen(1)
-
-        client = socket.socket()
-        client.setblocking(False)
-        try:
-            client.connect(('127.0.0.1', serverSocket.getsockname()[1]))
-        except socket.error as e:
-            assert e.args[0] in (errno.EINPROGRESS, errno.EWOULDBLOCK)
-        server, addr = serverSocket.accept()
-
-        # keep sockets alive during the test
-        self.connections.append(server)
-        self.connections.append(client)
-
-        return server.fileno(), client.fileno()
-
-    def w_make_pair(self):
-        import _multiprocessing
-
-        fd1, fd2 = self.socketpair()
-        rhandle = _multiprocessing.Connection(fd1, writable=False)
-        whandle = _multiprocessing.Connection(fd2, readable=False)
-        self.connections.append(rhandle)
-        self.connections.append(whandle)
-        return rhandle, whandle
-
-    def teardown_method(self, func):
-        # Work hard to close all sockets and connections now!
-        # since the fd is probably already closed, another unrelated
-        # part of the program will probably reuse it;
-        # And any object forgotten here will close it on destruction...
-        try:
-            w_connections = self.w_connections
-        except AttributeError:
-            return
-        space = self.space
-        for c in space.unpackiterable(w_connections):
-            if isinstance(c, W_Root):
-                space.call_method(c, "close")
-            else:
-                c.close()
-        space.delslice(w_connections, space.wrap(0), space.wrap(100))
-
-    def test_bad_fd(self):
-        import _multiprocessing
-
-        raises(IOError, _multiprocessing.Connection, -1)
-        raises(IOError, _multiprocessing.Connection, -15)
-
-    def test_byte_order(self):
-        import socket
-        if not 'fromfd' in dir(socket):
-            skip('No fromfd in socket')
-        # The exact format of net strings (length in network byte
-        # order) is important for interoperation with others
-        # implementations.
-        rhandle, whandle = self.make_pair()
-        whandle.send_bytes(b"abc")
-        whandle.send_bytes(b"defg")
-        sock = socket.fromfd(rhandle.fileno(),
-                             socket.AF_INET, socket.SOCK_STREAM)
-        data1 = sock.recv(7)
-        assert data1 == b'\x00\x00\x00\x03abc'
-        data2 = sock.recv(8)
-        assert data2 == b'\x00\x00\x00\x04defg'
-
-    def test_repr(self):
-        import _multiprocessing, os
-        fd = os.dup(1)     # closed by Connection.__del__
-        c = _multiprocessing.Connection(fd)
-        assert repr(c) == '<read-write Connection, handle %d>' % fd
-        if hasattr(_multiprocessing, 'PipeConnection'):
-            fd = os.dup(1)     # closed by PipeConnection.__del__
-            c = _multiprocessing.PipeConnection(fd)
-            assert repr(c) == '<read-write PipeConnection, handle %d>' % fd

From pypy.commits at gmail.com  Fri Feb  3 07:03:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 04:03:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: oops, fix
Message-ID: <58947187.977d1c0a.8f181.e4a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89910:d08bf476e7fc
Date: 2017-02-03 13:02 +0100
http://bitbucket.org/pypy/pypy/changeset/d08bf476e7fc/

Log:	oops, fix

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -15,6 +15,7 @@
 from pypy.interpreter.typedef import GetSetProperty, TypeDef
 
 RECURSIVE_MUTEX, SEMAPHORE = range(2)
+sys_platform = sys.platform
 
 if sys.platform == 'win32':
     from rpython.rlib import rwin32
@@ -331,6 +332,11 @@
         rgc.add_memory_pressure(SEM_T_SIZE)
         return sem
 
+    def reopen_semaphore(name):
+        sem = sem_open(name, 0, 0600, 0)
+        rgc.add_memory_pressure(SEM_T_SIZE)
+        return sem
+
     def delete_semaphore(handle):
         _sem_close_no_errno(handle)
 
@@ -523,10 +529,10 @@
     def rebuild(space, w_cls, w_handle, kind, maxvalue, w_name):
         name = space.str_or_None_w(w_name)
         #
-        if sys.platform != 'win32' and name is not None:
+        if sys_platform != 'win32' and name is not None:
             # like CPython, in this case ignore 'w_handle'
             try:
-                handle = create_semaphore(space, name, 0, maxvalue)
+                handle = reopen_semaphore(name)
             except OSError as e:
                 raise wrap_oserror(space, e)
         else:

From pypy.commits at gmail.com  Fri Feb  3 11:49:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 08:49:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Multiprocessing now fixed enough that I
 think we can run the tests nightly
Message-ID: <5894b494.da6d1c0a.a1b00.6527@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89911:987868f96726
Date: 2017-02-03 17:48 +0100
http://bitbucket.org/pypy/pypy/changeset/987868f96726/

Log:	Multiprocessing now fixed enough that I think we can run the tests
	nightly

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -304,10 +304,10 @@
     RegrTest('test_modulefinder.py'),
     RegrTest('test_msilib.py'),
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
-    RegrTest('test_multiprocessing_fork.py', skip="XXX: deadlocks?"),
-    RegrTest('test_multiprocessing_forkserver.py', skip="XXX: deadlocks?"),
-    RegrTest('test_multiprocessing_main_handling.py', skip="XXX: deadlocks?"),
-    RegrTest('test_multiprocessing_spawn.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_fork.py'),
+    RegrTest('test_multiprocessing_forkserver.py'),
+    RegrTest('test_multiprocessing_main_handling.py'),
+    RegrTest('test_multiprocessing_spawn.py'),
     RegrTest('test_netrc.py'),
     RegrTest('test_nis.py'),
     RegrTest('test_nntplib.py'),

From pypy.commits at gmail.com  Fri Feb  3 12:21:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 03 Feb 2017 09:21:26 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: A simple failing test for a numpy issue
Message-ID: <5894bc16.d699df0a.81733.18e5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89912:b60c94defbd0
Date: 2017-02-03 17:20 +0000
http://bitbucket.org/pypy/pypy/changeset/b60c94defbd0/

Log:	A simple failing test for a numpy issue

diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -79,6 +79,12 @@
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
 
+    def test_string_buf(self):
+        module = self.import_module(name='array')
+        arr = module.array('u', '123')
+        view = memoryview(arr)
+        assert view.itemsize == 4
+
     def test_subclass(self):
         import struct
         module = self.import_module(name='array')

From pypy.commits at gmail.com  Fri Feb  3 13:07:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 10:07:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5894c6f2.484a1c0a.a2edf.813c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89913:787209f76c53
Date: 2017-02-03 19:06 +0100
http://bitbucket.org/pypy/pypy/changeset/787209f76c53/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -421,7 +421,8 @@
         setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_allocated_length .+>)
         i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
-        --TICK--
+        # NB. we get threads because '_hashlib' uses ffi callback/def_extern
+        --THREAD-TICK--
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
         i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=<Calli . i EF=5 OS=110>)
         check_memory_error(i119)

From pypy.commits at gmail.com  Fri Feb  3 13:07:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 03 Feb 2017 10:07:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: accept 'py.test --pypy=~/some/path'
Message-ID: <5894c6f4.46bb1c0a.f706f.8866@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89914:0abed43feb07
Date: 2017-02-03 19:06 +0100
http://bitbucket.org/pypy/pypy/changeset/0abed43feb07/

Log:	accept 'py.test --pypy=~/some/path'

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -21,6 +21,7 @@
     def setup_class(cls):
         pypy_c = pytest.config.option.pypy_c or None
         if pypy_c is not None:
+            pypy_c = os.path.expanduser(pypy_c)
             assert os.path.exists(pypy_c), (
                 "--pypy specifies %r, which does not exist" % (pypy_c,))
             out = subprocess.check_output([pypy_c, '-c',

From pypy.commits at gmail.com  Fri Feb  3 13:09:18 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 03 Feb 2017 10:09:18 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Pass the itemsize information from the
 Py_buffer to the W_MemoryObject
Message-ID: <5894c74e.08061c0a.2afcb.d32a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89915:628879bbb5f4
Date: 2017-02-03 18:08 +0000
http://bitbucket.org/pypy/pypy/changeset/628879bbb5f4/

Log:	Pass the itemsize information from the Py_buffer to the
	W_MemoryObject

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -471,7 +471,7 @@
                             readonly=widen(pybuf.c_readonly),
                             releasebufferproc = rbp)
         fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        return space.newbuffer(buf, itemsize=buf.itemsize)
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -196,7 +196,7 @@
     def newseqiter(self, x):
         return w_some_obj()
 
-    def newbuffer(self, x):
+    def newbuffer(self, x, itemsize=1):
         return w_some_obj()
 
     def marshal_w(self, w_obj):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -295,7 +295,7 @@
         return W_LongObject.fromint(self, val)
 
     @specialize.argtype(1)
-    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type
         return W_LongObject.fromrarith_int(val)
 
     def newlong_from_rbigint(self, val):
@@ -350,8 +350,8 @@
     def newseqiter(self, w_obj):
         return W_SeqIterObject(w_obj)
 
-    def newbuffer(self, w_obj):
-        return W_MemoryView(w_obj)
+    def newbuffer(self, w_obj, itemsize=1):
+        return W_MemoryView(w_obj, itemsize=itemsize)
 
     def newbytes(self, s):
         return W_BytesObject(s)

From pypy.commits at gmail.com  Sat Feb  4 03:34:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 00:34:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Revert changes 0d92840e04ff and
 347b4430addd, breaks random user programs like 'nose'.
Message-ID: <589591f9.eb86df0a.3d2da.2185@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89916:3847916c649c
Date: 2017-02-04 09:33 +0100
http://bitbucket.org/pypy/pypy/changeset/3847916c649c/

Log:	Revert changes 0d92840e04ff and 347b4430addd, breaks random user
	programs like 'nose'.

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -198,21 +198,10 @@
 
     def getdict(self, space):
         if self.w_func_dict is None:
-            if not self.can_change_code:
-                raise oefmt(space.w_AttributeError,
-                            "cannot set extra attributes on built-in functions")
             self.w_func_dict = space.newdict(instance=True)
         return self.w_func_dict
 
-    def getdictvalue(self, space, attr):
-        if not self.can_change_code:
-            return None
-        return W_Root.getdictvalue(self, space, attr)
-
     def setdict(self, space, w_dict):
-        if not self.can_change_code:
-            raise oefmt(space.w_AttributeError,
-                        "cannot set __dict__ on built-in functions")
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError,
                         "setting function's dictionary to a non-dict")
@@ -671,7 +660,7 @@
         Function.__init__(self, func.space, func.code, func.w_func_globals,
                           func.defs_w, func.closure, func.name)
         self.w_doc = func.w_doc
-        #self.w_func_dict = func.w_func_dict---nowadays, always None
+        self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
 
     def descr_builtinfunction__new__(space, w_subtype):
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -98,14 +98,6 @@
         raises(TypeError, "dir.func_code = f.func_code")
         raises(TypeError, "list.append.im_func.func_code = f.func_code")
 
-    def test_write_extra_attributes_builtin_forbidden(self):
-        raises(AttributeError, "dir.abcd = 5")
-        raises(AttributeError, "list.append.im_func.efgh = 6")
-        raises(AttributeError, "dir.__dict__")
-        raises(AttributeError, "dir.__dict__ = {}")
-        c = all.__call__   # this should work
-        assert c([4, 5, 6]) is True
-
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")
         exec '''if 1:

From pypy.commits at gmail.com  Sat Feb  4 03:34:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 00:34:37 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5895921d.d18bdf0a.2b2da.1f76@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89917:9ff55469638c
Date: 2017-02-04 09:33 +0100
http://bitbucket.org/pypy/pypy/changeset/9ff55469638c/

Log:	hg merge default

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -207,21 +207,10 @@
 
     def getdict(self, space):
         if self.w_func_dict is None:
-            if not self.can_change_code:
-                raise oefmt(space.w_AttributeError,
-                            "cannot set extra attributes on built-in functions")
             self.w_func_dict = space.newdict(instance=True)
         return self.w_func_dict
 
-    def getdictvalue(self, space, attr):
-        if not self.can_change_code:
-            return None
-        return W_Root.getdictvalue(self, space, attr)
-
     def setdict(self, space, w_dict):
-        if not self.can_change_code:
-            raise oefmt(space.w_AttributeError,
-                        "cannot set __dict__ on built-in functions")
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError,
                         "setting function's dictionary to a non-dict")
@@ -706,7 +695,7 @@
                           func.defs_w, None, func.closure,
                           None, func.name)
         self.w_doc = func.w_doc
-        #self.w_func_dict = func.w_func_dict---nowadays, always None
+        self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
         self.w_kw_defs = func.w_kw_defs
 
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -163,14 +163,6 @@
         raises(TypeError, "dir.__code__ = f.__code__")
         raises(TypeError, "list.append.__code__ = f.__code__")
 
-    def test_write_extra_attributes_builtin_forbidden(self):
-        raises(AttributeError, "dir.abcd = 5")
-        raises(AttributeError, "list.append.im_func.efgh = 6")
-        raises(AttributeError, "dir.__dict__")
-        raises(AttributeError, "dir.__dict__ = {}")
-        c = all.__call__   # this should work
-        assert c([4, 5, 6]) is True
-
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")
         exec('''if 1:
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -65,6 +65,8 @@
     w_KeyError = W_TypeObject("KeyError")
     w_SystemExit = W_TypeObject("SystemExit")
     w_KeyboardInterrupt = W_TypeObject("KeyboardInterrupt")
+    w_RuntimeError = W_TypeObject("RuntimeError")
+    w_RecursionError = W_TypeObject("RecursionError")   # py3.5
     w_VisibleDeprecationWarning = W_TypeObject("VisibleDeprecationWarning")
     w_None = W_Root()
 

From pypy.commits at gmail.com  Sat Feb  4 08:07:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 05:07:32 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: update
Message-ID: <5895d214.da6d1c0a.67a1d.cc43@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5772:81216d358378
Date: 2017-02-04 14:07 +0100
http://bitbucket.org/pypy/extradoc/changeset/81216d358378/

Log:	update

diff --git a/planning/py3.5/milestone-2-progress.rst b/planning/py3.5/milestone-2-progress.rst
--- a/planning/py3.5/milestone-2-progress.rst
+++ b/planning/py3.5/milestone-2-progress.rst
@@ -12,6 +12,7 @@
 * At some point, review lib-python/conftest.py to remove the skips
   due to deadlocks (search for "XXX:").
   update 1: some have been reenabled already! (richard 30.sept)
+  update 2: most are gone (arigo 4 feb)
 
 * collections.py: ``OrderedDict`` should again be a thin wrapper over
   ``dict``.  The main pain point is ``move_to_end(last=False)``.  See

From pypy.commits at gmail.com  Sat Feb  4 08:32:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 05:32:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix the test
Message-ID: <5895d7fd.45aadf0a.33c59.84ac@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89918:0d6361cc38d8
Date: 2017-02-04 14:32 +0100
http://bitbucket.org/pypy/pypy/changeset/0d6361cc38d8/

Log:	fix the test

diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -102,16 +102,20 @@
         assert res == False
 
     def test_semaphore_rebuild(self):
-        from _multiprocessing import SemLock
+        from _multiprocessing import SemLock, sem_unlink
         kind = self.SEMAPHORE
         value = 1
         maxvalue = 1
-        sem = SemLock(kind, value, maxvalue, "4", unlink=True)
-
-        sem2 = SemLock._rebuild(sem.handle, kind, value, "92")
-        assert sem.handle != sem2.handle
-        sem2 = SemLock._rebuild(sem.handle, kind, value, None)
-        assert sem.handle == sem2.handle
+        sem = SemLock(kind, value, maxvalue, "4.2", unlink=False)
+        try:
+            sem2 = SemLock._rebuild(-1, kind, value, "4.2")
+            #assert sem.handle != sem2.handle---even though they come
+            # from different calls to sem_open(), on Linux at least,
+            # they are the same pointer
+            sem2 = SemLock._rebuild(sem.handle, kind, value, None)
+            assert sem.handle == sem2.handle
+        finally:
+            sem_unlink("4.2")
 
     def test_semaphore_contextmanager(self):
         from _multiprocessing import SemLock

From pypy.commits at gmail.com  Sat Feb  4 08:44:40 2017
From: pypy.commits at gmail.com (krono)
Date: Sat, 04 Feb 2017 05:44:40 -0800 (PST)
Subject: [pypy-commit] pypy default: Replace outdated,
 insecure greenlet link with readthedocs link
Message-ID: <5895dac8.d30f1c0a.79973.d350@mx.google.com>

Author: Tobias Pape <tobias at netshed.de>
Branch: 
Changeset: r89919:3c6cc7fcc1ed
Date: 2017-02-04 14:44 +0100
http://bitbucket.org/pypy/pypy/changeset/3c6cc7fcc1ed/

Log:	Replace outdated, insecure greenlet link with readthedocs link

diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -190,7 +190,7 @@
 from GC issues: if the program "forgets" an unfinished greenlet, it will
 always be collected at the next garbage collection.
 
-.. _documentation of the greenlets: http://packages.python.org/greenlet/
+.. _documentation of the greenlets: https://greenlet.readthedocs.io/
 
 
 Unimplemented features

From pypy.commits at gmail.com  Sat Feb  4 09:05:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 06:05:58 -0800 (PST)
Subject: [pypy-commit] cffi default: Make 'ffi.buffer' be the type of
 minibuffers returned by cffi, and
Message-ID: <5895dfc6.4a6b1c0a.2df5e.d8d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2877:d3a94b4aa932
Date: 2017-02-04 15:05 +0100
http://bitbucket.org/cffi/cffi/changeset/d3a94b4aa932/

Log:	Make 'ffi.buffer' be the type of minibuffers returned by cffi, and
	'ffi.buffer()' now calls the constructor.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6029,8 +6029,10 @@
     return result;
 }
 
-static PyObject *b_buffer(PyObject *self, PyObject *args, PyObject *kwds)
-{
+static PyObject *
+b_buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    /* this is the constructor of the type implemented in minibuffer.h */
     CDataObject *cd;
     Py_ssize_t size = -1;
     static char *keywords[] = {"cdata", "size", NULL};
@@ -6655,7 +6657,6 @@
     {"getcname", b_getcname, METH_VARARGS},
     {"string", (PyCFunction)b_string, METH_VARARGS | METH_KEYWORDS},
     {"unpack", (PyCFunction)b_unpack, METH_VARARGS | METH_KEYWORDS},
-    {"buffer", (PyCFunction)b_buffer, METH_VARARGS | METH_KEYWORDS},
     {"get_errno", b_get_errno, METH_NOARGS},
     {"set_errno", b_set_errno, METH_O},
     {"newp_handle", b_newp_handle, METH_VARARGS},
@@ -6959,6 +6960,10 @@
             INITERROR;
     }
 
+    Py_INCREF(&MiniBuffer_Type);
+    if (PyModule_AddObject(m, "buffer", (PyObject *)&MiniBuffer_Type) < 0)
+        INITERROR;
+
     init_cffi_tls();
     if (PyErr_Occurred())
         INITERROR;
diff --git a/c/cffi1_module.c b/c/cffi1_module.c
--- a/c/cffi1_module.c
+++ b/c/cffi1_module.c
@@ -45,6 +45,9 @@
         if (PyDict_SetItemString(FFI_Type.tp_dict, "CData",
                                  (PyObject *)&CData_Type) < 0)
             return -1;
+        if (PyDict_SetItemString(FFI_Type.tp_dict, "buffer",
+                                 (PyObject *)&MiniBuffer_Type) < 0)
+            return -1;
 
         for (i = 0; all_dlopen_flags[i].name != NULL; i++) {
             x = PyInt_FromLong(all_dlopen_flags[i].value);
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -475,19 +475,6 @@
 #define ffi_unpack  b_unpack     /* ffi_unpack() => b_unpack()
                                     from _cffi_backend.c */
 
-PyDoc_STRVAR(ffi_buffer_doc,
-"Return a read-write buffer object that references the raw C data\n"
-"pointed to by the given 'cdata'.  The 'cdata' must be a pointer or an\n"
-"array.  Can be passed to functions expecting a buffer, or directly\n"
-"manipulated with:\n"
-"\n"
-"    buf[:]          get a copy of it in a regular string, or\n"
-"    buf[idx]        as a single character\n"
-"    buf[:] = ...\n"
-"    buf[idx] = ...  change the content");
-
-#define ffi_buffer  b_buffer     /* ffi_buffer() => b_buffer()
-                                    from _cffi_backend.c */
 
 PyDoc_STRVAR(ffi_offsetof_doc,
 "Return the offset of the named field inside the given structure or\n"
@@ -1085,7 +1072,6 @@
 static PyMethodDef ffi_methods[] = {
  {"addressof",  (PyCFunction)ffi_addressof,  METH_VARARGS, ffi_addressof_doc},
  {"alignof",    (PyCFunction)ffi_alignof,    METH_O,       ffi_alignof_doc},
- {"buffer",     (PyCFunction)ffi_buffer,     METH_VKW,     ffi_buffer_doc},
  {"def_extern", (PyCFunction)ffi_def_extern, METH_VKW,     ffi_def_extern_doc},
  {"callback",   (PyCFunction)ffi_callback,   METH_VKW,     ffi_callback_doc},
  {"cast",       (PyCFunction)ffi_cast,       METH_VARARGS, ffi_cast_doc},
diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -238,6 +238,22 @@
 # define MINIBUF_TPFLAGS (Py_TPFLAGS_HAVE_GETCHARBUFFER | Py_TPFLAGS_HAVE_NEWBUFFER)
 #endif
 
+PyDoc_STRVAR(ffi_buffer_doc,
+"ffi.buffer(cdata):\n"
+"Return a read-write buffer object that references the raw C data\n"
+"pointed to by the given 'cdata'.  The 'cdata' must be a pointer or an\n"
+"array.  Can be passed to functions expecting a buffer, or directly\n"
+"manipulated with:\n"
+"\n"
+"    buf[:]          get a copy of it in a regular string, or\n"
+"    buf[idx]        as a single character\n"
+"    buf[:] = ...\n"
+"    buf[idx] = ...  change the content");
+
+static PyObject *            /* forward, implemented in _cffi_backend.c */
+b_buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
+
+
 static PyTypeObject MiniBuffer_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "_cffi_backend.buffer",
@@ -268,11 +284,25 @@
     &mb_as_buffer,                              /* tp_as_buffer */
     (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
         MINIBUF_TPFLAGS),                       /* tp_flags */
-    0,                                          /* tp_doc */
+    ffi_buffer_doc,                             /* tp_doc */
     (traverseproc)mb_traverse,                  /* tp_traverse */
     (inquiry)mb_clear,                          /* tp_clear */
     0,                                          /* tp_richcompare */
     offsetof(MiniBufferObj, mb_weakreflist),    /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    b_buffer_new,                               /* tp_new */
+    0,                                          /* tp_free */
 };
 
 static PyObject *minibuffer_new(char *data, Py_ssize_t size,
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2299,6 +2299,7 @@
     buf = buffer(c)
     assert repr(buf).startswith('<_cffi_backend.buffer object at 0x')
     assert bytes(buf) == b"hi there\x00"
+    assert type(buf) is buffer
     if sys.version_info < (3,):
         assert str(buf) == "hi there\x00"
         assert unicode(buf) == u+"hi there\x00"
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -93,6 +93,7 @@
             # ctypes backend: attach these constants to the instance
             self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
+        self.buffer = backend.buffer
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
@@ -316,18 +317,18 @@
         """
         return self._backend.unpack(cdata, length)
 
-    def buffer(self, cdata, size=-1):
-        """Return a read-write buffer object that references the raw C data
-        pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
-        an array.  Can be passed to functions expecting a buffer, or directly
-        manipulated with:
-
-            buf[:]          get a copy of it in a regular string, or
-            buf[idx]        as a single character
-            buf[:] = ...
-            buf[idx] = ...  change the content
-        """
-        return self._backend.buffer(cdata, size)
+   #def buffer(self, cdata, size=-1):
+   #    """Return a read-write buffer object that references the raw C data
+   #    pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
+   #    an array.  Can be passed to functions expecting a buffer, or directly
+   #    manipulated with:
+   #
+   #        buf[:]          get a copy of it in a regular string, or
+   #        buf[idx]        as a single character
+   #        buf[:] = ...
+   #        buf[idx] = ...  change the content
+   #    """
+   #    note that 'buffer' is a type, set on this instance by __init__
 
     def from_buffer(self, python_buffer):
         """Return a <cdata 'char[]'> that points to the data of the
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -172,6 +172,9 @@
 (This is similar to how ``str()`` gives inconsistent results on regular
 byte strings).  Use ``buf[:]`` instead.
 
+*New in version 1.10:* ``ffi.buffer`` is now the type of the returned
+buffer objects; ``ffi.buffer()`` actually calls the constructor.
+
 **ffi.from_buffer(python_buffer)**: return a ``<cdata 'char[]'>`` that
 points to the data of the given Python object, which must support the
 buffer interface.  This is the opposite of ``ffi.buffer()``.  It gives
@@ -334,6 +337,10 @@
 ``item``, ``length``, ``fields``, ``args``, ``result``, ``ellipsis``,
 ``abi``, ``elements`` and ``relements``.
 
+*New in version 1.10:* ``ffi.buffer`` is now `a type`__ as well.
+
+.. __: #ffi-buffer
+
 
 ffi.gc()
 ++++++++
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -43,6 +43,9 @@
   aside, ``ffi.string()`` no longer works on ``_Bool[]`` (but it never
   made much sense, as this function stops on the first zero).
 
+* ``ffi.buffer`` is now the name of cffi's buffer type, and
+  ``ffi.buffer()`` works like before but is the constructor of that type.
+
 
 v1.9
 ====
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1130,6 +1130,7 @@
             b = ffi.buffer(a)
         except NotImplementedError as e:
             py.test.skip(str(e))
+        assert type(b) is ffi.buffer
         content = b[:]
         assert len(content) == len(b) == 2
         if sys.byteorder == 'little':
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -233,6 +233,7 @@
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
     assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+    assert type(ffi.buffer(a)) is ffi.buffer
 
 def test_ffi_from_buffer():
     import array

From pypy.commits at gmail.com  Sat Feb  4 12:59:28 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 04 Feb 2017 09:59:28 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: hg merge py3.5
Message-ID: <58961680.91abdf0a.f1422.ca5b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89920:791a64ac3ea2
Date: 2017-02-04 17:56 +0000
http://bitbucket.org/pypy/pypy/changeset/791a64ac3ea2/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 5539 lines

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -706,7 +706,7 @@
 
     @classmethod
     def fromordinal(cls, n):
-        """Contruct a date from a proleptic Gregorian ordinal.
+        """Construct a date from a proleptic Gregorian ordinal.
 
         January 1 of year 1 is day 1.  Only the year, month and day are
         non-zero in the result.
@@ -1145,18 +1145,17 @@
     def __hash__(self):
         """Hash."""
         if self._hashcode == -1:
+            # PyPy: uses the same algo as _datetimemodule.c, which
+            # unlike the pure Python version always relies on the
+            # nondeterministic hash on strings
+            temp1 = timedelta(hours=self._hour,
+                              minutes=self._minute,
+                              seconds=self._second,
+                              microseconds=self._microsecond)
             tzoff = self.utcoffset()
-            if not tzoff:  # zero or None
-                self._hashcode = hash(self._getstate()[0])
-            else:
-                h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff,
-                              timedelta(hours=1))
-                assert not m % timedelta(minutes=1), "whole minute"
-                m //= timedelta(minutes=1)
-                if 0 <= h < 24:
-                    self._hashcode = hash(time(h, m, self.second, self.microsecond))
-                else:
-                    self._hashcode = hash((h, m, self.second, self.microsecond))
+            if tzoff:  # not zero, not None
+                temp1 -= tzoff
+            self._hashcode = hash(temp1)
         return self._hashcode
 
     # Conversion to string
diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -238,6 +238,12 @@
 # different than CPython's, always < 3000.  We get the 4-bytes string
 # here via a hack: MAGIC_NUMBER is set in the module from
 # module/_frozen_importlib/__init__.py before the module is executed.
+# FOR TESTS ONLY, we make it default to imp.get_magic().
+try:
+    MAGIC_NUMBER
+except NameError:
+    import imp
+    MAGIC_NUMBER = imp.get_magic()
 
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
diff --git a/lib-python/3/test/_test_multiprocessing.py b/lib-python/3/test/_test_multiprocessing.py
--- a/lib-python/3/test/_test_multiprocessing.py
+++ b/lib-python/3/test/_test_multiprocessing.py
@@ -2795,6 +2795,7 @@
             self.assertTrue((arena != narena and nstart == 0) or
                             (stop == nstart))
 
+    @test.support.cpython_only
     def test_free_from_gc(self):
         # Check that freeing of blocks by the garbage collector doesn't deadlock
         # (issue #12352).
@@ -2893,12 +2894,14 @@
         a = Foo()
         util.Finalize(a, conn.send, args=('a',))
         del a           # triggers callback for a
+        import gc; gc.collect()
 
         b = Foo()
         close_b = util.Finalize(b, conn.send, args=('b',))
         close_b()       # triggers callback for b
         close_b()       # does nothing because callback has already been called
         del b           # does nothing because callback has already been called
+        import gc; gc.collect()
 
         c = Foo()
         util.Finalize(c, conn.send, args=('c',))
diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -43,7 +43,7 @@
 
 def skip_unless_internalhash(test):
     """Skip decorator for tests that depend on SipHash24 or FNV"""
-    ok = sys.hash_info.algorithm in {"fnv", "siphash24", "rpython"}
+    ok = sys.hash_info.algorithm in {"fnv", "siphash24"}
     msg = "Requires SipHash24 or FNV"
     return test if ok else unittest.skip(msg)(test)
 
@@ -189,7 +189,6 @@
         stdout = out[1].strip()
         return int(stdout)
 
-    @impl_detail("PyPy does not support hash randomization", pypy=False)
     def test_randomized_hash(self):
         # two runs should return different hashes
         run1 = self.get_hash(self.repr_, seed='random')
@@ -197,10 +196,6 @@
         self.assertNotEqual(run1, run2)
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
-    if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -2
-    else:
-        EMPTY_STRING_HASH = 0
     repr_ = None
     repr_long = None
 
@@ -242,9 +237,9 @@
             [-283066365, -4576729883824601543, -271871407,
              -3927695501187247084],
         ],
-        'rpython': [
-            # This is for PyPy.  NOTE: PyPy does not support hash
-            # randomization for now, so the results don't depend on the seed.
+        'fnv-pypy': [
+            # This is for PyPy, whose fnv algorithm does not support hash
+            # randomization, so the results don't depend on the seed.
             # seed 0, 'abc'
             [-1600925533, 1453079729188098211, -1600925533,
              1453079729188098211],
@@ -273,6 +268,8 @@
         else:
             assert(sys.byteorder == 'big')
             platform = 3 if IS_64BIT else 2
+        if algorithm == 'fnv' and check_impl_detail(pypy=True):
+            algorithm = 'fnv-pypy'
         return self.known_hashes[algorithm][position][platform]
 
     def test_null_hash(self):
@@ -280,8 +277,7 @@
         known_hash_of_obj = self.get_expected_hash(0, 3)
 
         # Randomization is enabled by default:
-        if check_impl_detail(pypy=False):
-            self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
+        self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
 
         # It can also be disabled by setting the seed to 0:
         self.assertEqual(self.get_hash(self.repr_, seed=0), known_hash_of_obj)
@@ -307,11 +303,12 @@
     repr_long = repr('abcdefghijk')
     repr_ucs2 = repr('&#228;&#250;&#8721;&#8455;')
 
-    @impl_detail("hash('') == -2 on PyPy", pypy=False)
+    @impl_detail("hash('') != 0 on PyPy", pypy=False)
     @skip_unless_internalhash
     def test_empty_string(self):
         self.assertEqual(hash(""), 0)
 
+    @impl_detail("hash(ucs2) differs on PyPy if unichar is 4 bytes", pypy=False)
     @skip_unless_internalhash
     def test_ucs2_string(self):
         h = self.get_expected_hash(3, 6)
@@ -324,18 +321,20 @@
     repr_ = repr(b'abc')
     repr_long = repr(b'abcdefghijk')
 
+    @impl_detail("hash('') != 0 on PyPy", pypy=False)
     @skip_unless_internalhash
     def test_empty_string(self):
-        self.assertEqual(hash(b""), self.EMPTY_STRING_HASH)
+        self.assertEqual(hash(b""), 0)
 
 class MemoryviewHashRandomizationTests(StringlikeHashRandomizationTests,
                                        unittest.TestCase):
     repr_ = "memoryview(b'abc')"
     repr_long = "memoryview(b'abcdefghijk')"
 
+    @impl_detail("hash('') != 0 on PyPy", pypy=False)
     @skip_unless_internalhash
     def test_empty_string(self):
-        self.assertEqual(hash(memoryview(b"")), self.EMPTY_STRING_HASH)
+        self.assertEqual(hash(memoryview(b"")), 0)
 
 class DatetimeTests(HashRandomizationTests):
     def get_hash_command(self, repr_):
diff --git a/lib-python/3/test/test_signal.py b/lib-python/3/test/test_signal.py
--- a/lib-python/3/test/test_signal.py
+++ b/lib-python/3/test/test_signal.py
@@ -358,6 +358,8 @@
 
         assert_python_ok('-c', code)
 
+    @support.impl_detail("pypy writes the message to fd 2, not to sys.stderr",
+                         pypy=False)
     @unittest.skipIf(_testcapi is None, 'need _testcapi')
     def test_wakeup_write_error(self):
         # Issue #16105: write() errors in the C signal handler should not
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -261,7 +261,7 @@
     RegrTest('test_imaplib.py'),
     RegrTest('test_imghdr.py'),
     RegrTest('test_import'),
-    RegrTest('test_importlib', skip='XXX segfaults'),
+    RegrTest('test_importlib'),
     RegrTest('test_imp.py', core=True, usemodules='thread'),
     RegrTest('test_index.py'),
     RegrTest('test_inspect.py', usemodules="struct unicodedata"),
@@ -304,10 +304,10 @@
     RegrTest('test_modulefinder.py'),
     RegrTest('test_msilib.py'),
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
-    RegrTest('test_multiprocessing_fork.py', skip="XXX: deadlocks?"),
-    RegrTest('test_multiprocessing_forkserver.py', skip="XXX: deadlocks?"),
-    RegrTest('test_multiprocessing_main_handling.py', skip="XXX: deadlocks?"),
-    RegrTest('test_multiprocessing_spawn.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_fork.py'),
+    RegrTest('test_multiprocessing_forkserver.py'),
+    RegrTest('test_multiprocessing_main_handling.py'),
+    RegrTest('test_multiprocessing_spawn.py'),
     RegrTest('test_netrc.py'),
     RegrTest('test_nis.py'),
     RegrTest('test_nntplib.py'),
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -92,7 +92,11 @@
                 % (buf.nbytes, offset + size))
         raw_addr = buf._pypy_raw_address() + offset
         result = self.from_address(raw_addr)
-        result._ensure_objects()['ffffffff'] = obj
+        objects = result._ensure_objects()
+        if objects is not None:
+            objects['ffffffff'] = obj
+        else:   # case e.g. of a primitive type like c_int
+            result._objects = obj
         return result
 
     def from_buffer_copy(self, obj, offset=0):
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -201,6 +201,13 @@
                "issue, you can disable them here",
                default=True),
 
+    ChoiceOption("hash",
+                 "The hash function to use for strings: fnv from CPython 2.7"
+                 " or siphash24 from CPython >= 3.4",
+                 ["fnv", "siphash24"],
+                 default="siphash24",
+                 cmdline="--hash"),
+
     OptionDescription("std", "Standard Object Space Options", [
         BoolOption("withtproxy", "support transparent proxies",
                    default=True),
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -56,6 +56,9 @@
 .. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html
 .. _`RPython documentation`: http://rpython.readthedocs.org
 
+The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
+already installed. If translating on CPython, you need to install it, e.g. 
+using ``pip install cffi``.
 
 To build PyPy on Unix using the C translation backend, you need at least a C
 compiler and ``make`` installed. Further, some optional modules have additional
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -134,3 +134,12 @@
 rffi structures in cpyext can now be created by parsing simple C headers.
 Additionally, the cts object that holds the parsed information can act like
 cffi's ffi objects, with the methods cts.cast() and cts.gettype().
+
+.. branch: rpython-hash
+
+Don't freeze hashes in the translated pypy.  In practice, that means
+that we can now translate PyPy with the option --hash=siphash24 and get
+the same hashes as CPython 3.5, which can be randomized (in a
+crypographically good way).  It is the default in PyPy3.  The default of
+PyPy2 remains unchanged: there are user programs out there that depend
+on constant hashes (or even sometimes on specific hash results).
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -36,6 +36,7 @@
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
         w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+        hashfunc = space.config.objspace.hash
     else:
         w_initstdio = space.appexec([], """():
             return lambda unbuffered: None
@@ -46,6 +47,10 @@
             from rpython.jit.backend.hlinfo import highleveljitinfo
             highleveljitinfo.sys_executable = argv[0]
 
+        if hashfunc == "siphash24":
+            from rpython.rlib import rsiphash
+            rsiphash.enable_siphash24()
+
         #debug("entry point starting")
         #for arg in argv:
         #    debug(" argv -> " + arg)
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -728,8 +728,26 @@
             if IS_WINDOWS:
                 filename = filename.lower()
             if filename.endswith('.pyc') or filename.endswith('.pyo'):
+                # We don't actually load via SourcelessFileLoader
+                # because '__main__' must not be listed inside
+                # 'importlib._bootstrap._module_locks' (it deadlocks
+                # test_multiprocessing_main_handling.test_script_compiled)
+                from importlib._bootstrap_external import MAGIC_NUMBER
+                import marshal
                 loader = SourcelessFileLoader('__main__', filename)
-                args = (loader.load_module, loader.name)
+                mainmodule.__loader__ = loader
+                @hidden_applevel
+                def execfile(filename, namespace):
+                    with open(filename, 'rb') as f:
+                        if f.read(4) != MAGIC_NUMBER:
+                            raise RuntimeError("Bad magic number in .pyc file")
+                        if len(f.read(8)) != 8:
+                            raise RuntimeError("Truncated .pyc file")
+                        co = marshal.load(f)
+                    if type(co) is not type((lambda:0).__code__):
+                        raise RuntimeError("Bad code object in .pyc file")
+                    exec_(co, namespace)
+                args = (execfile, filename, mainmodule.__dict__)
             else:
                 filename = sys.argv[0]
                 for hook in sys.path_hooks:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -225,14 +225,14 @@
     def buffer_w(self, space, flags):
         if _does_override_buffer_w_ex(self.__class__):
             return self.buffer_w_ex(space, flags)[0]
-        return self._buffer(space, flags).buffer_w(space, flags)
+        return self.__buffer_w(space, flags).buffer_w(space, flags)
 
     def buffer_w_ex(self, space, flags):
         if _does_override_buffer_w(self.__class__):
             return self.buffer_w(space, flags), 'B', 1
-        return self._buffer(space, flags).buffer_w_ex(space, flags)
+        return self.__buffer_w(space, flags).buffer_w_ex(space, flags)
 
-    def _buffer(self, space, flags):
+    def __buffer_w(self, space, flags):
         if flags & space.BUF_WRITABLE:
             w_impl = space.lookup(self, '__wbuffer__')
         else:
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,7 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rlib import rstackovf
+from rpython.rlib import rstack, rstackovf
 
 from pypy.interpreter import debug
 
@@ -60,6 +60,7 @@
         "Check if this is an exception that should better not be caught."
         return (self.match(space, space.w_SystemExit) or
                 self.match(space, space.w_KeyboardInterrupt))
+        # note: an extra case is added in OpErrFmtNoArgs
 
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
@@ -512,6 +513,16 @@
     def _compute_value(self, space):
         return self._value.decode('utf-8')
 
+    def async(self, space):
+        # also matches a RuntimeError("maximum rec.") if the stack is
+        # still almost full, because in this case it might be a better
+        # idea to propagate the exception than eat it
+        if (self.w_type is space.w_RecursionError and
+            self._value == "maximum recursion depth exceeded" and
+            rstack.stack_almost_full()):
+            return True
+        return OperationError.async(self, space)
+
 @specialize.memo()
 def get_operr_class(valuefmt):
     try:
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -160,10 +160,8 @@
     def test_write_code_builtin_forbidden(self):
         def f(*args):
             return 42
-        if hasattr('dir', '__code__'):
-            # only on PyPy, CPython does not expose these attrs
-            raises(TypeError, "dir.__code__ = f.__code__")
-            raises(TypeError, "list().append.__func__.__code__ = f.__code__")
+        raises(TypeError, "dir.__code__ = f.__code__")
+        raises(TypeError, "list.append.__code__ = f.__code__")
 
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -88,6 +88,7 @@
         '_promote'                   : 'interp_magic._promote',
         'normalize_exc'             : 'interp_magic.normalize_exc',
         'StdErrPrinter'             : 'interp_stderrprinter.W_StdErrPrinter',
+        'stack_almost_full'         : 'interp_magic.stack_almost_full',
     }
 
     submodules = {
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -9,7 +9,7 @@
 from pypy.objspace.std.setobject import W_BaseSetObject
 from pypy.objspace.std.typeobject import MethodCache
 from pypy.objspace.std.mapdict import MapAttrCache
-from rpython.rlib import rposix, rgc
+from rpython.rlib import rposix, rgc, rstack
 
 
 def internal_repr(space, w_object):
@@ -176,3 +176,7 @@
     operr = OperationError(w_type, w_value, w_tb)
     operr.normalize_exception(space)
     return operr.get_w_value(space)
+
+def stack_almost_full(space):
+    """Return True if the stack is more than 15/16th full."""
+    return space.wrap(rstack.stack_almost_full())
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -23,13 +23,34 @@
 # ____________________________________________________________
 
 class UniqueCache:
+    for_testing = False    # set to True on the class level in test_c.py
+
     def __init__(self, space):
         self.ctvoid = None      # Cache for the 'void' type
         self.ctvoidp = None     # Cache for the 'void *' type
         self.ctchara = None     # Cache for the 'char[]' type
         self.primitives = {}    # Cache for {name: primitive_type}
         self.functions = []     # see _new_function_type()
-        self.for_testing = False
+        self.functions_packed = None     # only across translation
+
+    def _cleanup_(self):
+        import gc
+        assert self.functions_packed is None
+        # Note: a full PyPy translation may still have
+        # 'self.functions == []' at this point, possibly depending
+        # on details.  Code tested directly in test_ffi_obj
+        gc.collect()
+        funcs = []
+        for weakdict in self.functions:
+            funcs += weakdict._dict.values()
+        del self.functions[:]
+        self.functions_packed = funcs if len(funcs) > 0 else None
+
+    def unpack_functions(self):
+        for fct in self.functions_packed:
+            _record_function_type(self, fct)
+        self.functions_packed = None
+
 
 def _clean_cache(space):
     "NOT_RPYTHON"
@@ -622,7 +643,7 @@
     for w_arg in fargs:
         y = compute_identity_hash(w_arg)
         x = intmask((1000003 * x) ^ y)
-    x ^= (ellipsis - abi)
+    x ^= ellipsis + 2 * abi
     if unique_cache.for_testing:    # constant-folded to False in translation;
         x &= 3                      # but for test, keep only 2 bits of hash
     return x
@@ -646,6 +667,8 @@
     # one such dict, but in case of hash collision, there might be
     # more.
     unique_cache = space.fromcache(UniqueCache)
+    if unique_cache.functions_packed is not None:
+        unique_cache.unpack_functions()
     func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
     for weakdict in unique_cache.functions:
         ctype = weakdict.get(func_hash)
@@ -674,13 +697,18 @@
     #
     fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis, abi)
     unique_cache = space.fromcache(UniqueCache)
-    func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
+    _record_function_type(unique_cache, fct)
+    return fct
+
+def _record_function_type(unique_cache, fct):
+    from pypy.module._cffi_backend import ctypefunc
+    #
+    func_hash = _func_key_hash(unique_cache, fct.fargs, fct.ctitem,
+                               fct.ellipsis, fct.abi)
     for weakdict in unique_cache.functions:
         if weakdict.get(func_hash) is None:
-            weakdict.set(func_hash, fct)
             break
     else:
         weakdict = rweakref.RWeakValueDictionary(int, ctypefunc.W_CTypeFunc)
         unique_cache.functions.append(weakdict)
-        weakdict.set(func_hash, fct)
-    return fct
+    weakdict.set(func_hash, fct)
diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py
--- a/pypy/module/_cffi_backend/test/test_c.py
+++ b/pypy/module/_cffi_backend/test/test_c.py
@@ -37,6 +37,7 @@
     def setup_class(cls):
         testfuncs_w = []
         keepalive_funcs = []
+        UniqueCache.for_testing = True
 
         test_lib_c = tmpdir.join('_test_lib.c')
         src_test_lib_c = py.path.local(__file__).dirpath().join('_test_lib.c')
@@ -100,11 +101,12 @@
             _all_test_c.find_and_load_library = func
             _all_test_c._testfunc = testfunc
         """)
-        UniqueCache.for_testing = True
 
     def teardown_method(self, method):
+        _clean_cache(self.space)
+
+    def teardown_class(cls):
         UniqueCache.for_testing = False
-        _clean_cache(self.space)
 
 
 all_names = ', '.join(Module.interpleveldefs.keys())
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -1,5 +1,23 @@
+from pypy.module._cffi_backend import newtype
 from pypy.module._cffi_backend.newtype import _clean_cache
 
+
+class TestFFIObj:
+    spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
+
+    def teardown_method(self, meth):
+        _clean_cache(self.space)
+
+    def test_new_function_type_during_translation(self):
+        space = self.space
+        BInt = newtype.new_primitive_type(space, "int")
+        BFunc = newtype.new_function_type(space, space.wrap([BInt]), BInt)
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+        unique_cache = space.fromcache(newtype.UniqueCache)
+        unique_cache._cleanup_()
+        assert BFunc is newtype.new_function_type(space,space.wrap([BInt]),BInt)
+
+
 class AppTestFFIObj:
     spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
 
diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py
--- a/pypy/module/_multiprocessing/__init__.py
+++ b/pypy/module/_multiprocessing/__init__.py
@@ -5,7 +5,6 @@
 class Module(MixedModule):
 
     interpleveldefs = {
-        'Connection'      : 'interp_connection.W_FileConnection',
         'SemLock'         : 'interp_semaphore.W_SemLock',
         'sem_unlink'      : 'interp_semaphore.semaphore_unlink',
         'address_of_buffer' : 'interp_memory.address_of_buffer',
@@ -15,12 +14,5 @@
     }
 
     if sys.platform == 'win32':
-        interpleveldefs['PipeConnection'] = \
-            'interp_connection.W_PipeConnection'
         interpleveldefs['win32'] = 'interp_win32.win32_namespace(space)'
         del interpleveldefs['sem_unlink']
-
-    def init(self, space):
-        MixedModule.init(self, space)
-        from pypy.module._multiprocessing.interp_connection import State
-        space.fromcache(State).init(space)
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
deleted file mode 100644
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ /dev/null
@@ -1,547 +0,0 @@
-import sys
-from errno import EINTR
-
-from rpython.rlib import rpoll, rsocket
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rarithmetic import intmask
-from rpython.rtyper.lltypesystem import lltype, rffi
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
-from pypy.interpreter.gateway import (
-    WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
-from pypy.interpreter.typedef import GetSetProperty, TypeDef
-
-READABLE, WRITABLE = range(1, 3)
-PY_SSIZE_T_MAX = sys.maxint
-PY_SSIZE_T_MIN = -sys.maxint - 1
-
-class State(object):
-    def __init__(self, space):
-        pass
-
-    def init(self, space):
-        w_builtins = space.getbuiltinmodule('builtins')
-        w_module = space.call_method(
-            w_builtins, '__import__', space.wrap("multiprocessing"))
-        self.w_BufferTooShort = space.getattr(w_module, space.wrap("BufferTooShort"))
-
-        self.w_picklemodule = space.call_method(
-            w_builtins, '__import__', space.wrap("pickle"))
-
-def BufferTooShort(space, w_data):
-    state = space.fromcache(State)
-    if not we_are_translated() and not hasattr(state, 'w_BufferTooShort'):
-        state.init(space)   # xxx for test/test_connection.py
-    w_BufferTooShort = state.w_BufferTooShort
-    return OperationError(w_BufferTooShort, w_data)
-
-def w_handle(space, handle):
-    return space.wrap(rffi.cast(rffi.INTPTR_T, handle))
-
-
-class W_BaseConnection(W_Root):
-    BUFFER_SIZE = 1024
-    buffer = lltype.nullptr(rffi.CCHARP.TO)
-
-    def __init__(self, space, flags):
-        self.flags = flags
-        self.buffer = lltype.malloc(rffi.CCHARP.TO, self.BUFFER_SIZE,
-                                    flavor='raw')
-        self.register_finalizer(space)
-
-    def _finalize_(self):
-        buf = self.buffer
-        if buf:
-            self.buffer = lltype.nullptr(rffi.CCHARP.TO)
-            lltype.free(buf, flavor='raw')
-        try:
-            self.do_close()
-        except OSError:
-            pass
-
-    # Abstract methods
-    def do_close(self):
-        raise NotImplementedError
-    def is_valid(self):
-        return False
-    def do_send_string(self, space, buf, offset, size):
-        raise NotImplementedError
-    def do_recv_string(self, space, buflength, maxlength):
-        raise NotImplementedError
-    def do_poll(self, space, timeout):
-        raise NotImplementedError
-
-    def close(self):
-        self.do_close()
-
-    def closed_get(self, space):
-        return space.newbool(not self.is_valid())
-    def readable_get(self, space):
-        return space.newbool(bool(self.flags & READABLE))
-    def writable_get(self, space):
-        return space.newbool(bool(self.flags & WRITABLE))
-
-    def _repr(self, space, handle):
-        index = self.flags - 1
-        conn_type = [u"read-only", u"write-only", u"read-write"][index]
-        return space.wrap(u"<%s %s, handle %d>" % (
-                conn_type, space.type(self).getname(space), handle))
-
-    def descr_repr(self, space):
-        raise NotImplementedError
-
-    def _check_readable(self, space):
-        if not self.flags & READABLE:
-            raise oefmt(space.w_IOError, "connection is write-only")
-    def _check_writable(self, space):
-        if not self.flags & WRITABLE:
-            raise oefmt(space.w_IOError, "connection is read-only")
-
-    @unwrap_spec(offset='index', size='index')
-    def send_bytes(self, space, w_buf, offset=0, size=PY_SSIZE_T_MIN):
-        buf = space.getarg_w('s*', w_buf).as_str()
-        length = len(buf)
-        self._check_writable(space)
-        if offset < 0:
-            raise oefmt(space.w_ValueError, "offset is negative")
-        if length < offset:
-            raise oefmt(space.w_ValueError, "buffer length < offset")
-
-        if size == PY_SSIZE_T_MIN:
-            size = length - offset
-        elif size < 0:
-            raise oefmt(space.w_ValueError, "size is negative")
-        elif offset + size > length:
-            raise oefmt(space.w_ValueError, "buffer length > offset + size")
-
-        self.do_send_string(space, buf, offset, size)
-
-    @unwrap_spec(maxlength='index')
-    def recv_bytes(self, space, maxlength=PY_SSIZE_T_MAX):
-        self._check_readable(space)
-        if maxlength < 0:
-            raise oefmt(space.w_ValueError, "maxlength < 0")
-
-        res, newbuf = self.do_recv_string(
-            space, self.BUFFER_SIZE, maxlength)
-        try:
-            if newbuf:
-                return space.newbytes(rffi.charpsize2str(newbuf, res))
-            else:
-                return space.newbytes(rffi.charpsize2str(self.buffer, res))
-        finally:
-            if newbuf:
-                rffi.free_charp(newbuf)
-
-    @unwrap_spec(offset='index')
-    def recv_bytes_into(self, space, w_buffer, offset=0):
-        rwbuffer = space.writebuf_w(w_buffer)
-        length = rwbuffer.getlength()
-
-        res, newbuf = self.do_recv_string(
-            space, length - offset, PY_SSIZE_T_MAX)
-        try:
-            if newbuf:
-                raise BufferTooShort(space, space.newbytes(
-                    rffi.charpsize2str(newbuf, res)))
-            rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res))
-        finally:
-            if newbuf:
-                rffi.free_charp(newbuf)
-
-        return space.wrap(res)
-
-    def send(self, space, w_obj):
-        self._check_writable(space)
-
-        w_picklemodule = space.fromcache(State).w_picklemodule
-        w_protocol = space.getattr(
-            w_picklemodule, space.wrap("HIGHEST_PROTOCOL"))
-        w_pickled = space.call_method(
-            w_picklemodule, "dumps", w_obj, w_protocol)
-
-        buf = space.str_w(w_pickled)
-        self.do_send_string(space, buf, 0, len(buf))
-
-    def recv(self, space):
-        self._check_readable(space)
-
-        res, newbuf = self.do_recv_string(
-            space, self.BUFFER_SIZE, PY_SSIZE_T_MAX)
-        try:
-            if newbuf:
-                w_received = space.newbytes(rffi.charpsize2str(newbuf, res))
-            else:
-                w_received = space.newbytes(rffi.charpsize2str(self.buffer, res))
-        finally:
-            if newbuf:
-                rffi.free_charp(newbuf)
-
-        w_builtins = space.getbuiltinmodule('builtins')
-        w_picklemodule = space.fromcache(State).w_picklemodule
-        w_unpickled = space.call_method(
-            w_picklemodule, "loads", w_received)
-
-        return w_unpickled
-
-    @unwrap_spec(w_timeout=WrappedDefault(0.0))
-    def poll(self, space, w_timeout):
-        self._check_readable(space)
-        if space.is_w(w_timeout, space.w_None):
-            timeout = -1.0 # block forever
-        else:
-            timeout = space.float_w(w_timeout)
-            if timeout < 0.0:
-                timeout = 0.0
-        return space.newbool(self.do_poll(space, timeout))
-
-W_BaseConnection.typedef = TypeDef(
-    'BaseConnection',
-    __repr__ = interpindirect2app(W_BaseConnection.descr_repr),
-    closed = GetSetProperty(W_BaseConnection.closed_get),
-    readable = GetSetProperty(W_BaseConnection.readable_get),
-    writable = GetSetProperty(W_BaseConnection.writable_get),
-
-    send_bytes = interp2app(W_BaseConnection.send_bytes),
-    recv_bytes = interp2app(W_BaseConnection.recv_bytes),
-    recv_bytes_into = interp2app(W_BaseConnection.recv_bytes_into),
-    send = interp2app(W_BaseConnection.send),
-    recv = interp2app(W_BaseConnection.recv),
-    poll = interp2app(W_BaseConnection.poll),
-    close = interp2app(W_BaseConnection.close),
-    )
-
-class W_FileConnection(W_BaseConnection):
-    INVALID_HANDLE_VALUE = -1
-    fd = INVALID_HANDLE_VALUE
-
-    if sys.platform == 'win32':
-        def WRITE(self, data):
-            from rpython.rlib._rsocket_rffi import send, geterrno
-            length = send(self.fd, data, len(data), 0)
-            if length < 0:
-                raise WindowsError(geterrno(), "send")
-            return length
-        def READ(self, size):
-            from rpython.rlib._rsocket_rffi import socketrecv, geterrno
-            with rffi.scoped_alloc_buffer(size) as buf:
-                length = socketrecv(self.fd, buf.raw, buf.size, 0)
-                if length < 0:
-                    raise WindowsError(geterrno(), "recv")
-                return buf.str(length)
-        def CLOSE(self):
-            from rpython.rlib._rsocket_rffi import socketclose
-            socketclose(self.fd)
-    else:
-        def WRITE(self, data):
-            import os
-            return os.write(self.fd, data)
-        def READ(self, length):
-            import os
-            return os.read(self.fd, length)
-        def CLOSE(self):
-            import os
-            try:
-                os.close(self.fd)
-            except OSError:
-                pass
-
-    def __init__(self, space, fd, flags):
-        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
-            raise oefmt(space.w_IOError, "invalid handle %d", fd)
-        W_BaseConnection.__init__(self, space, flags)
-        self.fd = fd
-
-    @unwrap_spec(fd=int, readable=bool, writable=bool)
-    def descr_new_file(space, w_subtype, fd, readable=True, writable=True):
-        flags = (readable and READABLE) | (writable and WRITABLE)
-
-        self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, space, fd, flags)
-        return space.wrap(self)
-
-    def descr_repr(self, space):
-        return self._repr(space, self.fd)
-
-    def fileno(self, space):
-        return space.wrap(self.fd)
-
-    def is_valid(self):
-        return self.fd != self.INVALID_HANDLE_VALUE
-
-    def do_close(self):
-        if self.is_valid():
-            self.CLOSE()
-            self.fd = self.INVALID_HANDLE_VALUE
-
-    def do_send_string(self, space, buf, offset, size):
-        # Since str2charp copies the buf anyway, always combine the
-        # "header" and the "body" of the message and send them at once.
-        message = lltype.malloc(rffi.CCHARP.TO, size + 4, flavor='raw')
-        try:
-            length = rffi.r_uint(rsocket.htonl(
-                    rffi.cast(lltype.Unsigned, size)))
-            rffi.cast(rffi.UINTP, message)[0] = length
-            i = size - 1
-            while i >= 0:
-                message[4 + i] = buf[offset + i]
-                i -= 1
-            self._sendall(space, message, size + 4)
-        finally:
-            lltype.free(message, flavor='raw')
-
-    def do_recv_string(self, space, buflength, maxlength):
-        with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1) as length_ptr:
-            self._recvall(space, rffi.cast(rffi.CCHARP, length_ptr), 4)
-            length = intmask(rsocket.ntohl(
-                    rffi.cast(lltype.Unsigned, length_ptr[0])))
-        if length > maxlength: # bad message, close connection
-            self.flags &= ~READABLE
-            if self.flags == 0:
-                self.close()
-            raise oefmt(space.w_IOError, "bad message length")
-
-        if length <= buflength:
-            self._recvall(space, self.buffer, length)
-            return length, lltype.nullptr(rffi.CCHARP.TO)
-        else:
-            newbuf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
-            self._recvall(space, newbuf, length)
-            return length, newbuf
-
-    def _sendall(self, space, message, size):
-        while size > 0:
-            # XXX inefficient
-            data = rffi.charpsize2str(message, size)
-            try:
-                count = self.WRITE(data)
-            except OSError as e:
-                if e.errno == EINTR:
-                    space.getexecutioncontext().checksignals()
-                    continue
-                raise wrap_oserror(space, e)
-            size -= count
-            message = rffi.ptradd(message, count)
-
-    def _recvall(self, space, buf, length):
-        length = intmask(length)
-        remaining = length
-        while remaining > 0:
-            try:
-                data = self.READ(remaining)
-            except OSError as e:
-                if e.errno == EINTR:
-                    space.getexecutioncontext().checksignals()
-                    continue
-                raise wrap_oserror(space, e)
-            count = len(data)
-            if count == 0:
-                if remaining == length:
-                    raise OperationError(space.w_EOFError, space.w_None)
-                else:
-                    raise oefmt(space.w_IOError,
-                                "got end of file during message")
-            # XXX inefficient
-            for i in range(count):
-                buf[i] = data[i]
-            remaining -= count
-            buf = rffi.ptradd(buf, count)
-
-    if sys.platform == 'win32':
-        def _check_fd(self):
-            return self.fd >= 0
-    else:
-        def _check_fd(self):
-            return self.fd >= 0 and self.fd < rpoll.FD_SETSIZE
-
-    def do_poll(self, space, timeout):
-        if not self._check_fd():
-            raise oefmt(space.w_IOError, "handle out of range in select()")
-        r, w, e = rpoll.select([self.fd], [], [], timeout, handle_eintr=True)
-        return bool(r)
-
-W_FileConnection.typedef = TypeDef(
-    '_multiprocessing.Connection', W_BaseConnection.typedef,
-    __new__ = interp2app(W_FileConnection.descr_new_file.im_func),
-    fileno = interp2app(W_FileConnection.fileno),
-)
-
-class W_PipeConnection(W_BaseConnection):
-    if sys.platform == 'win32':
-        from rpython.rlib.rwin32 import INVALID_HANDLE_VALUE
-
-    def __init__(self, space, handle, flags):
-        W_BaseConnection.__init__(self, space, flags)
-        self.handle = handle
-
-    @unwrap_spec(readable=bool, writable=bool)
-    def descr_new_pipe(space, w_subtype, w_handle, readable=True,
-                       writable=True):
-        from pypy.module._multiprocessing.interp_win32 import handle_w
-        handle = handle_w(space, w_handle)
-        flags = (readable and READABLE) | (writable and WRITABLE)
-
-        self = space.allocate_instance(W_PipeConnection, w_subtype)
-        W_PipeConnection.__init__(self, space, handle, flags)
-        return space.wrap(self)
-
-    def descr_repr(self, space):
-        return self._repr(space, rffi.cast(rffi.INTPTR_T, self.handle))
-
-    def is_valid(self):
-        return self.handle != self.INVALID_HANDLE_VALUE
-
-    def fileno(self, space):
-        return w_handle(space, self.handle)
-
-    def do_close(self):
-        from rpython.rlib.rwin32 import CloseHandle
-        if self.is_valid():
-            CloseHandle(self.handle)
-            self.handle = self.INVALID_HANDLE_VALUE
-
-    def do_send_string(self, space, buf, offset, size):
-        from pypy.module._multiprocessing.interp_win32 import (
-            _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
-        from rpython.rlib import rwin32
-
-        with rffi.scoped_view_charp(buf) as charp:
-            written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                        flavor='raw')
-            try:
-                result = _WriteFile(
-                    self.handle, rffi.ptradd(charp, offset),
-                    size, written_ptr, rffi.NULL)
-
-                if (result == 0 and
-                    rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
-                    raise oefmt(space.w_ValueError,
-                                "Cannot send %d bytes over connection", size)
-            finally:
-                lltype.free(written_ptr, flavor='raw')
-
-    def do_recv_string(self, space, buflength, maxlength):
-        from pypy.module._multiprocessing.interp_win32 import (
-            _ReadFile, _PeekNamedPipe, ERROR_BROKEN_PIPE, ERROR_MORE_DATA)
-        from rpython.rlib import rwin32
-        from pypy.interpreter.error import wrap_windowserror
-
-        read_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                 flavor='raw')
-        left_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                 flavor='raw')
-        try:
-            result = _ReadFile(self.handle,
-                               self.buffer, min(self.BUFFER_SIZE, buflength),
-                               read_ptr, rffi.NULL)
-            if result:
-                return intmask(read_ptr[0]), lltype.nullptr(rffi.CCHARP.TO)
-
-            err = rwin32.GetLastError_saved()
-            if err == ERROR_BROKEN_PIPE:
-                raise OperationError(space.w_EOFError, space.w_None)
-            elif err != ERROR_MORE_DATA:
-                raise wrap_windowserror(space, WindowsError(err, "_ReadFile"))
-
-            # More data...
-            if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
-                                  lltype.nullptr(rwin32.LPDWORD.TO),
-                                  lltype.nullptr(rwin32.LPDWORD.TO),
-                                  left_ptr):
-                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-
-            length = intmask(read_ptr[0] + left_ptr[0])
-            if length > maxlength: # bad message, close connection
-                self.flags &= ~READABLE
-                if self.flags == 0:
-                    self.close()
-                raise oefmt(space.w_IOError, "bad message length")
-
-            newbuf = lltype.malloc(rffi.CCHARP.TO, length + 1, flavor='raw')
-            for i in range(read_ptr[0]):
-                newbuf[i] = self.buffer[i]
-
-            result = _ReadFile(self.handle,
-                               rffi.ptradd(newbuf, read_ptr[0]), left_ptr[0],
-                               read_ptr, rffi.NULL)
-            if not result:
-                rffi.free_charp(newbuf)
-                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-
-            assert read_ptr[0] == left_ptr[0]
-            return length, newbuf
-        finally:
-            lltype.free(read_ptr, flavor='raw')
-            lltype.free(left_ptr, flavor='raw')
-
-    def do_poll(self, space, timeout):
-        from pypy.module._multiprocessing.interp_win32 import (
-            _PeekNamedPipe, _GetTickCount, _Sleep)
-        from rpython.rlib import rwin32
-        from pypy.interpreter.error import wrap_windowserror
-        bytes_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                 flavor='raw')
-        try:
-            if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
-                                  lltype.nullptr(rwin32.LPDWORD.TO),
-                                  bytes_ptr,
-                                  lltype.nullptr(rwin32.LPDWORD.TO)):
-                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-            bytes = bytes_ptr[0]
-        finally:
-            lltype.free(bytes_ptr, flavor='raw')
-
-        if timeout == 0.0:
-            return bytes > 0
-
-        block = timeout < 0
-        if not block:
-            # XXX does not check for overflow
-            deadline = intmask(_GetTickCount()) + int(1000 * timeout + 0.5)
-        else:
-            deadline = 0
-
-        _Sleep(0)
-
-        delay = 1
-        while True:
-            bytes_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
-                                     flavor='raw')
-            try:
-                if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
-                                      lltype.nullptr(rwin32.LPDWORD.TO),
-                                      bytes_ptr,
-                                      lltype.nullptr(rwin32.LPDWORD.TO)):
-                    raise wrap_windowserror(space,
-                                            rwin32.lastSavedWindowsError())
-                bytes = bytes_ptr[0]
-            finally:
-                lltype.free(bytes_ptr, flavor='raw')
-
-            if bytes > 0:
-                return True
-
-            if not block:
-                now = intmask(_GetTickCount())
-                if now > deadline:
-                    return False
-                diff = deadline - now
-                if delay > diff:
-                    delay = diff
-            else:
-                delay += 1
-
-            if delay >= 20:
-                delay = 20
-            _Sleep(delay)
-
-            # check for signals
-            # PyErr_CheckSignals()
-
-if sys.platform == 'win32':
-    W_PipeConnection.typedef = TypeDef(
-        '_multiprocessing.PipeConnection', W_BaseConnection.typedef,
-        __new__ = interp2app(W_PipeConnection.descr_new_pipe.im_func),
-        fileno = interp2app(W_PipeConnection.fileno),
-    )
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -13,9 +13,9 @@
 from pypy.interpreter.error import oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import GetSetProperty, TypeDef
-from pypy.module._multiprocessing.interp_connection import w_handle
 
 RECURSIVE_MUTEX, SEMAPHORE = range(2)
+sys_platform = sys.platform
 
 if sys.platform == 'win32':
     from rpython.rlib import rwin32
@@ -216,9 +216,10 @@
 
     def semaphore_unlink(space, w_name):
         name = space.str_w(w_name)
-        res = _sem_unlink(name)
-        if res < 0:
-            raise oefmt(space.w_OSError, "sem unlink failed with errno: %d", rposix.get_saved_errno())
+        try:
+            sem_unlink(name)
+        except OSError as e:
+            raise wrap_oserror(space, e)
 
 class CounterState:
     def __init__(self, space):
@@ -331,6 +332,11 @@
         rgc.add_memory_pressure(SEM_T_SIZE)
         return sem
 
+    def reopen_semaphore(name):
+        sem = sem_open(name, 0, 0600, 0)
+        rgc.add_memory_pressure(SEM_T_SIZE)
+        return sem
+
     def delete_semaphore(handle):
         _sem_close_no_errno(handle)
 
@@ -454,7 +460,8 @@
         return space.newint(self.maxvalue)
 
     def handle_get(self, space):
-        return w_handle(space, self.handle)
+        h = rffi.cast(rffi.INTPTR_T, self.handle)
+        return space.wrap(h)
 
     def get_count(self, space):
         return space.wrap(self.count)
@@ -521,8 +528,18 @@
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue, w_name):
         name = space.str_or_None_w(w_name)
+        #
+        if sys_platform != 'win32' and name is not None:
+            # like CPython, in this case ignore 'w_handle'
+            try:
+                handle = reopen_semaphore(name)
+            except OSError as e:
+                raise wrap_oserror(space, e)
+        else:
+            handle = handle_w(space, w_handle)
+        #
         self = space.allocate_instance(W_SemLock, w_cls)
-        self.__init__(space, handle_w(space, w_handle), kind, maxvalue, name)
+        self.__init__(space, handle, kind, maxvalue, name)
         return space.wrap(self)
 
     def enter(self, space):
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
deleted file mode 100644
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ /dev/null
@@ -1,200 +0,0 @@
-import py
-import sys
-from pypy.interpreter.gateway import interp2app, W_Root
-
-class TestImport:
-    def test_simple(self):
-        from pypy.module._multiprocessing import interp_connection
-        from pypy.module._multiprocessing import interp_semaphore
-
-class AppTestBufferTooShort:
-    spaceconfig = {'usemodules': ['_multiprocessing', 'thread', 'signal',
-                                  'select', 'struct', 'binascii',
-                                  '_posixsubprocess']}
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_rawffi')
-    else:
-        spaceconfig['usemodules'].append('fcntl')
-
-
-    def setup_class(cls):
-        if cls.runappdirect:
-            def raiseBufferTooShort(self, data):
-                import multiprocessing
-                raise multiprocessing.BufferTooShort(data)
-            cls.w_raiseBufferTooShort = raiseBufferTooShort
-        else:
-            from pypy.module._multiprocessing import interp_connection
-            def raiseBufferTooShort(space, w_data):
-                raise interp_connection.BufferTooShort(space, w_data)
-            cls.w_raiseBufferTooShort = cls.space.wrap(
-                interp2app(raiseBufferTooShort))
-
-    def test_exception(self):
-        import multiprocessing
-        try:
-            self.raiseBufferTooShort("data")
-        except multiprocessing.BufferTooShort as e:
-            assert isinstance(e, multiprocessing.ProcessError)
-            assert e.args == ("data",)
-
-class BaseConnectionTest(object):
-    def test_connection(self):
-        rhandle, whandle = self.make_pair()
-
-        whandle.send_bytes(b"abc")
-        assert rhandle.recv_bytes(100) == b"abc"
-
-        obj = [1, 2.0, "hello"]
-        whandle.send(obj)
-        obj2 = rhandle.recv()
-        assert obj == obj2
-
-    def test_poll(self):
-        rhandle, whandle = self.make_pair()
-
-        assert rhandle.poll() == False
-        assert rhandle.poll(1) == False
-        whandle.send(1)
-        import time; time.sleep(0.1)  # give it time to arrive :-)
-        assert rhandle.poll() == True
-        assert rhandle.poll(None) == True
-        assert rhandle.recv() == 1
-        assert rhandle.poll() == False
-        raises(IOError, whandle.poll)
-
-    def test_read_into(self):
-        import array, multiprocessing
-        rhandle, whandle = self.make_pair()
-
-        obj = [1, 2.0, "hello"]
-        whandle.send(obj)
-        buffer = array.array('b', [0]*10)
-        raises(multiprocessing.BufferTooShort, rhandle.recv_bytes_into, buffer)
-        assert rhandle.readable
-
-class AppTestWinpipeConnection(BaseConnectionTest):
-    spaceconfig = {
-        "usemodules": [
-            '_multiprocessing', 'thread', 'signal', 'struct', 'array',
-            'itertools', '_socket', 'binascii',
-        ]
-    }
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_rawffi')
-
-    def setup_class(cls):
-        if sys.platform != "win32":
-            py.test.skip("win32 only")
-
-        if not cls.runappdirect:
-            space = cls.space
-            # stubs for some modules,
-            # just for multiprocessing to import correctly on Windows
-            w_modules = space.sys.get('modules')
-            space.setitem(w_modules, space.wrap('msvcrt'), space.sys)
-        else:
-            import _multiprocessing
-
-    def w_make_pair(self):
-        import multiprocessing
-
-        return multiprocessing.Pipe(duplex=False)
-
-
-class AppTestSocketConnection(BaseConnectionTest):
-    spaceconfig = {
-        "usemodules": [
-            '_multiprocessing', 'thread', 'signal', 'struct', 'array',
-            '_socket', 'binascii', 'select', '_posixsubprocess']
-    }
-    if sys.platform == 'win32':
-        spaceconfig['usemodules'].append('_rawffi')
-    else:
-        spaceconfig['usemodules'].append('fcntl')
-
-    def setup_class(cls):
-        cls.w_connections = cls.space.newlist([])
-
-    def w_socketpair(self):
-        "A socket.socketpair() that works on Windows"
-        import errno
-        import socket
-
-        serverSocket = socket.socket()
-        serverSocket.bind(('127.0.0.1', 0))
-        serverSocket.listen(1)
-
-        client = socket.socket()
-        client.setblocking(False)
-        try:
-            client.connect(('127.0.0.1', serverSocket.getsockname()[1]))
-        except socket.error as e:
-            assert e.args[0] in (errno.EINPROGRESS, errno.EWOULDBLOCK)
-        server, addr = serverSocket.accept()
-
-        # keep sockets alive during the test
-        self.connections.append(server)
-        self.connections.append(client)
-
-        return server.fileno(), client.fileno()
-
-    def w_make_pair(self):
-        import _multiprocessing
-
-        fd1, fd2 = self.socketpair()
-        rhandle = _multiprocessing.Connection(fd1, writable=False)
-        whandle = _multiprocessing.Connection(fd2, readable=False)
-        self.connections.append(rhandle)
-        self.connections.append(whandle)
-        return rhandle, whandle
-
-    def teardown_method(self, func):
-        # Work hard to close all sockets and connections now!
-        # since the fd is probably already closed, another unrelated
-        # part of the program will probably reuse it;
-        # And any object forgotten here will close it on destruction...
-        try:
-            w_connections = self.w_connections
-        except AttributeError:
-            return
-        space = self.space
-        for c in space.unpackiterable(w_connections):
-            if isinstance(c, W_Root):
-                space.call_method(c, "close")
-            else:
-                c.close()
-        space.delslice(w_connections, space.wrap(0), space.wrap(100))
-
-    def test_bad_fd(self):
-        import _multiprocessing
-
-        raises(IOError, _multiprocessing.Connection, -1)
-        raises(IOError, _multiprocessing.Connection, -15)
-
-    def test_byte_order(self):
-        import socket
-        if not 'fromfd' in dir(socket):
-            skip('No fromfd in socket')
-        # The exact format of net strings (length in network byte
-        # order) is important for interoperation with others
-        # implementations.
-        rhandle, whandle = self.make_pair()
-        whandle.send_bytes(b"abc")
-        whandle.send_bytes(b"defg")
-        sock = socket.fromfd(rhandle.fileno(),
-                             socket.AF_INET, socket.SOCK_STREAM)
-        data1 = sock.recv(7)
-        assert data1 == b'\x00\x00\x00\x03abc'
-        data2 = sock.recv(8)
-        assert data2 == b'\x00\x00\x00\x04defg'
-
-    def test_repr(self):
-        import _multiprocessing, os
-        fd = os.dup(1)     # closed by Connection.__del__
-        c = _multiprocessing.Connection(fd)
-        assert repr(c) == '<read-write Connection, handle %d>' % fd
-        if hasattr(_multiprocessing, 'PipeConnection'):
-            fd = os.dup(1)     # closed by PipeConnection.__del__
-            c = _multiprocessing.PipeConnection(fd)
-            assert repr(c) == '<read-write PipeConnection, handle %d>' % fd
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -22,10 +22,13 @@
     @py.test.mark.skipif("sys.platform == 'win32'")
     def test_sem_unlink(self):
         from _multiprocessing import sem_unlink
+        import errno
         try:
             sem_unlink("non-existent")
-        except OSError:
-            pass
+        except OSError as e:
+            assert e.errno in (errno.ENOENT, errno.EINVAL)
+        else:
+            assert 0, "should have raised"
 
     def test_semaphore(self):
         from _multiprocessing import SemLock
@@ -99,14 +102,20 @@
         assert res == False
 
     def test_semaphore_rebuild(self):
-        from _multiprocessing import SemLock
+        from _multiprocessing import SemLock, sem_unlink
         kind = self.SEMAPHORE
         value = 1
         maxvalue = 1
-        sem = SemLock(kind, value, maxvalue, "4", unlink=True)
-
-        sem2 = SemLock._rebuild(sem.handle, kind, value, "10")
-        assert sem.handle == sem2.handle
+        sem = SemLock(kind, value, maxvalue, "4.2", unlink=False)
+        try:
+            sem2 = SemLock._rebuild(-1, kind, value, "4.2")
+            #assert sem.handle != sem2.handle---even though they come
+            # from different calls to sem_open(), on Linux at least,
+            # they are the same pointer
+            sem2 = SemLock._rebuild(sem.handle, kind, value, None)
+            assert sem.handle == sem2.handle
+        finally:
+            sem_unlink("4.2")
 
     def test_semaphore_contextmanager(self):
         from _multiprocessing import SemLock
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -194,6 +194,15 @@
         W_WeakrefBase.__init__(self, space, w_obj, w_callable)
         self.w_hash = None
 
+    def _cleanup_(self):
+        # When a prebuilt weakref is frozen inside a translation, if
+        # this weakref has got an already-cached w_hash, then throw it
+        # away.  That's because the hash value will change after
+        # translation.  It will be recomputed the first time we ask for
+        # it.  Note that such a frozen weakref, if not dead, will point
+        # to a frozen object, so it will never die.
+        self.w_hash = None
+
     def descr__init__weakref(self, space, w_obj, w_callable=None,
                              __args__=None):
         if __args__.arguments_w:
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -141,12 +141,120 @@
                                         hints={'nolength': True}))
 
 class W_ArrayBase(W_Root):
-    _attrs_ = ('space', 'len', 'allocated', '_lifeline_') # no buffer
+    _attrs_ = ('space', 'len', 'allocated', '_lifeline_', '_buffer')
 
     def __init__(self, space):
         self.space = space
         self.len = 0
         self.allocated = 0
+        self._buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+    @rgc.must_be_light_finalizer
+    def __del__(self):
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+
+    def setlen(self, size, zero=False, overallocate=True):
+        if size > 0:
+            if size > self.allocated or size < self.allocated / 2:
+                if overallocate:
+                    if size < 9:
+                        some = 3
+                    else:
+                        some = 6
+                    some += size >> 3
+                else:
+                    some = 0
+                self.allocated = size + some
+                byte_size = self.allocated * self.itemsize
+                if zero:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True, zero=True)
+                else:
+                    new_buffer = lltype.malloc(
+                        rffi.CCHARP.TO, byte_size, flavor='raw',
+                        add_memory_pressure=True)
+                    copy_bytes = min(size, self.len) * self.itemsize
+                    rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer),
+                                  rffi.cast(rffi.VOIDP, self._buffer),
+                                  copy_bytes)
+            else:
+                self.len = size
+                return
+        else:
+            assert size == 0
+            self.allocated = 0
+            new_buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+        if self._buffer:
+            lltype.free(self._buffer, flavor='raw')
+        self._buffer = new_buffer
+        self.len = size
+
+    def _fromiterable(self, w_seq):
+        # used by fromsequence().
+        # a more careful case if w_seq happens to be a very large
+        # iterable: don't copy the items into some intermediate list
+        w_iterator = self.space.iter(w_seq)
+        tp = self.space.type(w_iterator)
+        while True:
+            unpack_driver.jit_merge_point(selfclass=self.__class__,
+                                          tp=tp, self=self,
+                                          w_iterator=w_iterator)
+            space = self.space
+            try:
+                w_item = space.next(w_iterator)
+            except OperationError as e:
+                if not e.match(space, space.w_StopIteration):
+                    raise
+                break  # done
+            self.descr_append(space, w_item)
+
+    def _charbuf_start(self):
+        return self._buffer
+
+    def _buffer_as_unsigned(self):
+        return rffi.cast(lltype.Unsigned, self._buffer)
+
+    def _charbuf_stop(self):
+        keepalive_until_here(self)
+
+    def delitem(self, space, i, j):
+        if i < 0:
+            i += self.len
+        if i < 0:
+            i = 0
+        if j < 0:
+            j += self.len
+        if j < 0:
+            j = 0
+        if j > self.len:
+            j = self.len
+        if i >= j:
+            return None
+        oldbuffer = self._buffer
+        self._buffer = lltype.malloc(rffi.CCHARP.TO,
+            (self.len - (j - i)) * self.itemsize, flavor='raw',
+            add_memory_pressure=True)
+        if i:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, self._buffer),
+                rffi.cast(rffi.VOIDP, oldbuffer),
+                i * self.itemsize
+            )
+        if j < self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                                  i * self.itemsize)),
+                rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer,
+                                                  j * self.itemsize)),
+                (self.len - j) * self.itemsize
+            )
+        self.len -= j - i
+        self.allocated = self.len
+        if oldbuffer:
+            lltype.free(oldbuffer, flavor='raw')
 
     def buffer_w_ex(self, space, flags):
         return ArrayBuffer(self, False), self.typecode, self.itemsize
@@ -170,14 +278,24 @@
 
         Return number of occurrences of x in the array.
         """
-        raise NotImplementedError
+        cnt = 0
+        for i in range(self.len):
+            # XXX jitdriver
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_val)):
+                cnt += 1
+        return space.wrap(cnt)
 
     def descr_index(self, space, w_x):
         """ index(x)
 
         Return index of first occurrence of x in the array.
         """
-        raise NotImplementedError
+        for i in range(self.len):
+            w_item = self.w_getitem(space, i)
+            if space.is_true(space.eq(w_item, w_x)):
+                return space.wrap(i)
+        raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
     def descr_reverse(self, space):
         """ reverse()
@@ -191,7 +309,8 @@
 
         Remove the first occurrence of x in the array.
         """
-        raise NotImplementedError
+        w_idx = self.descr_index(space, w_val)
+        self.descr_pop(space, space.int_w(w_idx))
 
     @unwrap_spec(i=int)
     def descr_pop(self, space, i=-1):
@@ -507,16 +626,102 @@
         return space.newseqiter(self)
 
     def descr_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        a = self.constructor(space)
+        a.setlen(self.len + w_other.len, overallocate=False)
+        if self.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, a._buffer),
+                rffi.cast(rffi.VOIDP, self._buffer),
+                self.len * self.itemsize
+            )
+        if w_other.len:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(a._buffer,
+                                             self.len * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                w_other.len * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_inplace_add(self, space, w_other):
-        raise NotImplementedError
+        if (not isinstance(w_other, W_ArrayBase)
+                or w_other.typecode != self.typecode):
+            return space.w_NotImplemented
+        oldlen = self.len
+        otherlen = w_other.len
+        self.setlen(oldlen + otherlen)
+        if otherlen:
+            rffi.c_memcpy(
+                rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+                                             oldlen * self.itemsize)),
+                rffi.cast(rffi.VOIDP, w_other._buffer),
+                otherlen * self.itemsize
+            )
+        keepalive_until_here(self)
+        keepalive_until_here(w_other)
+        return self
+
+    def _mul_helper(self, space, w_repeat, is_inplace):
+        try:
+            repeat = space.getindex_w(w_repeat, space.w_OverflowError)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
+        if is_inplace:
+            a = self
+            start = 1
+        else:
+            a = self.constructor(space)
+            start = 0
+        if repeat <= start:
+            if repeat <= 0:
+                a.setlen(0, overallocate=False)
+            return a
+        oldlen = self.len
+        try:
+            newlen = ovfcheck(oldlen * repeat)
+        except OverflowError:
+            raise MemoryError
+        #
+        srcbuf = self._buffer
+        srcsize = self.len * self.itemsize
+        for i in range(srcsize):
+            if srcbuf[i] != '\x00':
+                break
+        else:
+            # the source is entirely zero: initialize the target
+            # with zeroes too
+            a.setlen(newlen, zero=True, overallocate=False)
+            return a
+        #
+        a.setlen(newlen, overallocate=False)
+        srcbuf = self._buffer   # reload this, in case self is a
+        if oldlen == 1:
+            self._repeat_single_item(a, start, repeat)
+        else:
+            dstbuf = a._buffer
+            if start == 1:
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+            for r in range(start, repeat):
+                rffi.c_memcpy(rffi.cast(rffi.VOIDP, dstbuf),
+                              rffi.cast(rffi.VOIDP, srcbuf),
+                              srcsize)
+                dstbuf = rffi.ptradd(dstbuf, srcsize)
+        keepalive_until_here(self)
+        keepalive_until_here(a)
+        return a
 
     def descr_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, False)
 
     def descr_inplace_mul(self, space, w_repeat):
-        raise NotImplementedError
+        return self._mul_helper(space, w_repeat, True)
 
     def descr_radd(self, space, w_other):
         return self.descr_add(space, w_other)
@@ -600,6 +805,7 @@
         self.itemtype = itemtype
         self.bytes = rffi.sizeof(itemtype)
         self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
+        self.arrayptrtype = lltype.Ptr(self.arraytype)
         self.unwrap, _, self.convert = unwrap.partition('.')
         self.signed = signed
         self.canoverflow = canoverflow
@@ -694,22 +900,21 @@
         return self.array._charbuf_start()
 
 
+unpack_driver = jit.JitDriver(name='unpack_array',
+                              greens=['selfclass', 'tp'],
+                              reds=['self', 'w_iterator'])
+
 def make_array(mytype):
     W_ArrayBase = globals()['W_ArrayBase']
 
-    unpack_driver = jit.JitDriver(name='unpack_array',
-                                  greens=['tp'],
-                                  reds=['self', 'w_iterator'])
-
     class W_Array(W_ArrayBase):
         itemsize = mytype.bytes
         typecode = mytype.typecode
 
-        _attrs_ = ('space', 'len', 'allocated', '_lifeline_', 'buffer')
+        _attrs_ = W_ArrayBase._attrs_
 
-        def __init__(self, space):
-            W_ArrayBase.__init__(self, space)
-            self.buffer = lltype.nullptr(mytype.arraytype)
+        def get_buffer(self):
+            return rffi.cast(mytype.arrayptrtype, self._buffer)
 
         def item_w(self, w_item):
             space = self.space
@@ -765,46 +970,6 @@
                                          self.space.wrap(msg))
             return result
 
-        @rgc.must_be_light_finalizer
-        def __del__(self):
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-
-        def setlen(self, size, zero=False, overallocate=True):
-            if size > 0:
-                if size > self.allocated or size < self.allocated / 2:
-                    if overallocate:
-                        if size < 9:
-                            some = 3
-                        else:
-                            some = 6
-                        some += size >> 3
-                    else:
-                        some = 0
-                    self.allocated = size + some
-                    if zero:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True, zero=True)
-                    else:
-                        new_buffer = lltype.malloc(
-                            mytype.arraytype, self.allocated, flavor='raw',
-                            add_memory_pressure=True)
-                        for i in range(min(size, self.len)):
-                            new_buffer[i] = self.buffer[i]
-                else:
-                    self.len = size
-                    return
-            else:
-                assert size == 0
-                self.allocated = 0
-                new_buffer = lltype.nullptr(mytype.arraytype)
-
-            if self.buffer:
-                lltype.free(self.buffer, flavor='raw')
-            self.buffer = new_buffer
-            self.len = size
-
         def fromsequence(self, w_seq):
             space = self.space
             oldlen = self.len
@@ -820,20 +985,21 @@
             if lst is not None:
                 self.setlen(oldlen + len(lst))
                 try:
-                    buf = self.buffer
+                    buf = self.get_buffer()
                     for num in lst:
                         buf[newlen] = self.item_from_int_or_float(num)
                         newlen += 1
                 except OperationError:
                     self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             # this is the common case: w_seq is a list or a tuple
             lst_w = space.listview_no_unpack(w_seq)
             if lst_w is not None:
                 self.setlen(oldlen + len(lst_w))
-                buf = self.buffer
+                buf = self.get_buffer()
                 try:
                     for w_num in lst_w:
                         # note: self.item_w() might invoke arbitrary code.
@@ -844,30 +1010,14 @@
                         buf[newlen] = self.item_w(w_num)
                         newlen += 1
                 except OperationError:
-                    if buf == self.buffer:
+                    if buf == self.get_buffer():
                         self.setlen(newlen)
                     raise
+                keepalive_until_here(self)
                 return
 
             self._fromiterable(w_seq)
 
-        def _fromiterable(self, w_seq):
-            # a more careful case if w_seq happens to be a very large
-            # iterable: don't copy the items into some intermediate list
-            w_iterator = self.space.iter(w_seq)
-            tp = self.space.type(w_iterator)
-            while True:
-                unpack_driver.jit_merge_point(tp=tp, self=self,
-                                              w_iterator=w_iterator)
-                space = self.space
-                try:
-                    w_item = space.next(w_iterator)
-                except OperationError as e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    break  # done
-                self.descr_append(space, w_item)
-
         def extend(self, w_iterable, accept_different_array=False):
             space = self.space
             if isinstance(w_iterable, W_Array):
@@ -875,11 +1025,14 @@
                 new = w_iterable.len
                 self.setlen(self.len + new)
                 i = 0
+                buf = self.get_buffer()
+                srcbuf = w_iterable.get_buffer()
                 while i < new:
                     if oldlen + i >= self.len:
                         self.setlen(oldlen + i + 1)
-                    self.buffer[oldlen + i] = w_iterable.buffer[i]
+                    buf[oldlen + i] = srcbuf[i]
                     i += 1
+                keepalive_until_here(w_iterable)
                 self.setlen(oldlen + i)
             elif (not accept_different_array
                   and isinstance(w_iterable, W_ArrayBase)):
@@ -888,17 +1041,9 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def _charbuf_start(self):
-            return rffi.cast(rffi.CCHARP, self.buffer)
-
-        def _buffer_as_unsigned(self):
-            return rffi.cast(lltype.Unsigned, self.buffer)
-
-        def _charbuf_stop(self):
+        def w_getitem(self, space, idx):
+            item = self.get_buffer()[idx]
             keepalive_until_here(self)
-
-        def w_getitem(self, space, idx):
-            item = self.buffer[idx]
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
             elif mytype.typecode == 'f':
@@ -916,29 +1061,16 @@
             x = self.item_w(w_x)
             index = self.len
             self.setlen(index + 1)
-            self.buffer[index] = x
+            self.get_buffer()[index] = x
+            keepalive_until_here(self)
 
         # List interface
-        def descr_count(self, space, w_val):
-            cnt = 0
-            for i in range(self.len):
-                # XXX jitdriver
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    cnt += 1
-            return space.wrap(cnt)
-
-        def descr_index(self, space, w_val):
-            for i in range(self.len):
-                w_item = self.w_getitem(space, i)
-                if space.is_true(space.eq(w_item, w_val)):
-                    return space.wrap(i)
-            raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
         def descr_reverse(self, space):
-            b = self.buffer
+            b = self.get_buffer()
             for i in range(self.len / 2):
                 b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i]
+            keepalive_until_here(self)
 
         def descr_pop(self, space, i):
             if i < 0:
@@ -946,16 +1078,14 @@
             if i < 0 or i >= self.len:
                 raise oefmt(space.w_IndexError, "pop index out of range")
             w_val = self.w_getitem(space, i)
+            b = self.get_buffer()
             while i < self.len - 1:
-                self.buffer[i] = self.buffer[i + 1]
+                b[i] = b[i + 1]
                 i += 1
+            keepalive_until_here(self)
             self.setlen(self.len - 1)
             return w_val
 
-        def descr_remove(self, space, w_val):
-            w_idx = self.descr_index(space, w_val)
-            self.descr_pop(space, space.int_w(w_idx))
-
         def descr_insert(self, space, idx, w_val):
             if idx < 0:
                 idx += self.len
@@ -967,10 +1097,12 @@
             val = self.item_w(w_val)
             self.setlen(self.len + 1)
             i = self.len - 1
+            b = self.get_buffer()
             while i > idx:
-                self.buffer[i] = self.buffer[i - 1]
+                b[i] = b[i - 1]
                 i -= 1

From pypy.commits at gmail.com  Sat Feb  4 13:01:18 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 04 Feb 2017 10:01:18 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: document branch
Message-ID: <589616ee.c69bdf0a.b3cdd.cb0f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89921:663c79d098fd
Date: 2017-02-04 18:00 +0000
http://bitbucket.org/pypy/pypy/changeset/663c79d098fd/

Log:	document branch

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -36,3 +36,7 @@
 .. branch: py3.5-time
 
 .. branch: py3.5-ssl
+
+.. branch: PEP393
+
+Implement some level of compatibility with PEP 393 APIs.

From pypy.commits at gmail.com  Sat Feb  4 13:10:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 04 Feb 2017 10:10:17 -0800 (PST)
Subject: [pypy-commit] pypy PEP393: Close branch PEP393
Message-ID: <58961909.48301c0a.908f3.2e4d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PEP393
Changeset: r89922:aebbc8b7f1af
Date: 2017-02-04 18:09 +0000
http://bitbucket.org/pypy/pypy/changeset/aebbc8b7f1af/

Log:	Close branch PEP393


From pypy.commits at gmail.com  Sat Feb  4 13:10:34 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 04 Feb 2017 10:10:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Merged in PEP393 (pull request #513)
Message-ID: <5896191a.9d711c0a.875ba.2fc6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89923:c36e0e41d514
Date: 2017-02-04 18:09 +0000
http://bitbucket.org/pypy/pypy/changeset/c36e0e41d514/

Log:	Merged in PEP393 (pull request #513)

	PEP 393

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -36,3 +36,7 @@
 .. branch: py3.5-time
 
 .. branch: py3.5-ssl
+
+.. branch: PEP393
+
+Implement some level of compatibility with PEP 393 APIs.
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,12 +7,318 @@
 
 #include <cpyext_unicodeobject.h>
 
-PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs);
-PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...);
+/* Fast access macros */
+#ifndef Py_LIMITED_API
 
+#define PyUnicode_WSTR_LENGTH(op) \
+    (PyUnicode_IS_COMPACT_ASCII(op) ?                  \
+     ((PyASCIIObject*)op)->length :                    \
+     ((PyCompactUnicodeObject*)op)->wstr_length)
+
+/* Returns the deprecated Py_UNICODE representation's size in code units
+   (this includes surrogate pairs as 2 units).
+   If the Py_UNICODE representation is not available, it will be computed
+   on request.  Use PyUnicode_GET_LENGTH() for the length in code points. */
+
+#define PyUnicode_GET_SIZE(op)                       \
+    (assert(PyUnicode_Check(op)),                    \
+     (((PyASCIIObject *)(op))->wstr) ?               \
+      PyUnicode_WSTR_LENGTH(op) :                    \
+      ((void)PyUnicode_AsUnicode((PyObject *)(op)),  \
+       assert(((PyASCIIObject *)(op))->wstr),        \
+       PyUnicode_WSTR_LENGTH(op)))
+
+#define PyUnicode_GET_DATA_SIZE(op) \
+    (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE)
+
+/* Alias for PyUnicode_AsUnicode().  This will create a wchar_t/Py_UNICODE
+   representation on demand.  Using this macro is very inefficient now,
+   try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
+   use PyUnicode_WRITE() and PyUnicode_READ(). */
+
+#define PyUnicode_AS_UNICODE(op) \
+    (assert(PyUnicode_Check(op)), \
+     (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \
+      PyUnicode_AsUnicode((PyObject *)(op)))
+
+#define PyUnicode_AS_DATA(op) \
+    ((const char *)(PyUnicode_AS_UNICODE(op)))
+
+
+/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */
+
+/* Values for PyASCIIObject.state: */
+
+/* Interning state. */
+#define SSTATE_NOT_INTERNED 0
+#define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+
+/* Return true if the string contains only ASCII characters, or 0 if not. The
+   string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be
+   ready. */
+#define PyUnicode_IS_ASCII(op)                   \
+    (assert(PyUnicode_Check(op)),                \
+     assert(PyUnicode_IS_READY(op)),             \
+     ((PyASCIIObject*)op)->state.ascii)
+
+/* Return true if the string is compact or 0 if not.
+   No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT(op) \
+    (((PyASCIIObject*)(op))->state.compact)
+
+/* Return true if the string is a compact ASCII string (use PyASCIIObject
+   structure), or 0 if not.  No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT_ASCII(op)                 \
+    (((PyASCIIObject*)op)->state.ascii && PyUnicode_IS_COMPACT(op))
+
+enum PyUnicode_Kind {
+/* String contains only wstr byte characters.  This is only possible
+   when the string was created with a legacy API and _PyUnicode_Ready()
+   has not been called yet.  */
+    PyUnicode_WCHAR_KIND = 0,
+/* Return values of the PyUnicode_KIND() macro: */
+    PyUnicode_1BYTE_KIND = 1,
+    PyUnicode_2BYTE_KIND = 2,
+    PyUnicode_4BYTE_KIND = 4
+};
+
+/* Return pointers to the canonical representation cast to unsigned char,
+   Py_UCS2, or Py_UCS4 for direct character access.
+   No checks are performed, use PyUnicode_KIND() before to ensure
+   these will work correctly. */
+
+#define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op))
+#define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op))
+#define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op))
+
+/* Return one of the PyUnicode_*_KIND values defined above. */
+#define PyUnicode_KIND(op) \
+    (assert(PyUnicode_Check(op)), \
+     assert(PyUnicode_IS_READY(op)),            \
+     ((PyASCIIObject *)(op))->state.kind)
+
+/* Return a void pointer to the raw unicode buffer. */
+#define _PyUnicode_COMPACT_DATA(op)                     \
+    (PyUnicode_IS_ASCII(op) ?                   \
+     ((void*)((PyASCIIObject*)(op) + 1)) :              \
+     ((void*)((PyCompactUnicodeObject*)(op) + 1)))
+
+#define _PyUnicode_NONCOMPACT_DATA(op)                  \
+    (assert(((PyUnicodeObject*)(op))->data),        \
+     ((((PyUnicodeObject *)(op))->data)))
+
+#define PyUnicode_DATA(op) \
+    (assert(PyUnicode_Check(op)), \
+     PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) :   \
+     _PyUnicode_NONCOMPACT_DATA(op))
+
+/* In the access macros below, "kind" may be evaluated more than once.
+   All other macro parameters are evaluated exactly once, so it is safe
+   to put side effects into them (such as increasing the index). */
+
+/* Write into the canonical representation, this macro does not do any sanity
+   checks and is intended for usage in loops.  The caller should cache the
+   kind and data pointers obtained from other macro calls.
+   index is the index in the string (starts at 0) and value is the new
+   code point value which should be written to that location. */
+#define PyUnicode_WRITE(kind, data, index, value) \
+    do { \
+        switch ((kind)) { \
+        case PyUnicode_1BYTE_KIND: { \
+            ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \
+            break; \
+        } \
+        case PyUnicode_2BYTE_KIND: { \
+            ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \
+            break; \
+        } \
+        default: { \
+            assert((kind) == PyUnicode_4BYTE_KIND); \
+            ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \
+        } \
+        } \
+    } while (0)
+
+/* Read a code point from the string's canonical representation.  No checks
+   or ready calls are performed. */
+#define PyUnicode_READ(kind, data, index) \
+    ((Py_UCS4) \
+    ((kind) == PyUnicode_1BYTE_KIND ? \
+        ((const Py_UCS1 *)(data))[(index)] : \
+        ((kind) == PyUnicode_2BYTE_KIND ? \
+            ((const Py_UCS2 *)(data))[(index)] : \
+            ((const Py_UCS4 *)(data))[(index)] \
+        ) \
+    ))
+
+/* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it
+   calls PyUnicode_KIND() and might call it twice.  For single reads, use
+   PyUnicode_READ_CHAR, for multiple consecutive reads callers should
+   cache kind and use PyUnicode_READ instead. */
+#define PyUnicode_READ_CHAR(unicode, index) \
+    (assert(PyUnicode_Check(unicode)),          \
+     assert(PyUnicode_IS_READY(unicode)),       \
+     (Py_UCS4)                                  \
+        (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \
+            ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \
+            (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \
+                ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \
+                ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \
+            ) \
+        ))
+
+/* Returns the length of the unicode string. The caller has to make sure that
+   the string has it's canonical representation set before calling
+   this macro.  Call PyUnicode_(FAST_)Ready to ensure that. */
+#define PyUnicode_GET_LENGTH(op)                \
+    (assert(PyUnicode_Check(op)),               \
+     assert(PyUnicode_IS_READY(op)),            \
+     ((PyASCIIObject *)(op))->length)
+
+
+/* Fast check to determine whether an object is ready. Equivalent to
+   PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */
+
+#define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready)
+
+/* PyUnicode_READY() does less work than _PyUnicode_Ready() in the best
+   case.  If the canonical representation is not yet set, it will still call
+   _PyUnicode_Ready().
+   Returns 0 on success and -1 on errors. */
+#define PyUnicode_READY(op)                        \
+    (assert(PyUnicode_Check(op)),                       \
+     (PyUnicode_IS_READY(op) ?                          \
+      0 : _PyUnicode_Ready((PyObject *)(op))))
+
+/* Return a maximum character value which is suitable for creating another
+   string based on op.  This is always an approximation but more efficient
+   than iterating over the string. */
+#define PyUnicode_MAX_CHAR_VALUE(op) \
+    (assert(PyUnicode_IS_READY(op)),                                    \
+     (PyUnicode_IS_ASCII(op) ?                                          \
+      (0x7f) :                                                          \
+      (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ?                     \
+       (0xffU) :                                                        \
+       (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ?                    \
+        (0xffffU) :                                                     \
+        (0x10ffffU)))))
+
+#endif
+
+/* --- Constants ---------------------------------------------------------- */
+
+/* This Unicode character will be used as replacement character during
+   decoding if the errors argument is set to "replace". Note: the
+   Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
+   Unicode 3.0. */
+
+#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD)
+
+/* === Public API ========================================================= */
+
+PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(
+    const char *format,   /* ASCII-encoded string  */
+    va_list vargs
+    );
+PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(
+    const char *format,   /* ASCII-encoded string  */
+    ...
+    );
+
+/* Use only if you know it's a string */
+#define PyUnicode_CHECK_INTERNED(op) \
+    (((PyASCIIObject *)(op))->state.interned)
+
+/* --- wchar_t support for platforms which support it --------------------- */
+
+#ifdef HAVE_WCHAR_H
+
+/* Convert the Unicode object to a wide character string. The output string
+   always ends with a nul character. If size is not NULL, write the number of
+   wide characters (excluding the null character) into *size.
+
+   Returns a buffer allocated by PyMem_Malloc() (use PyMem_Free() to free it)
+   on success. On error, returns NULL, *size is undefined and raises a
+   MemoryError. */
+
+PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString(
+    PyObject *unicode,          /* Unicode object */
+    Py_ssize_t *size            /* number of characters of the result */
+    );
+
+#endif
+
+/* === Builtin Codecs =====================================================
+
+   Many of these APIs take two arguments encoding and errors. These
+   parameters encoding and errors have the same semantics as the ones
+   of the builtin str() API.
+
+   Setting encoding to NULL causes the default encoding (UTF-8) to be used.
+
+   Error handling is set by errors which may also be set to NULL
+   meaning to use the default handling defined for the codec. Default
+   error handling for all builtin codecs is "strict" (ValueErrors are
+   raised).
+
+   The codecs all use a similar interface. Only deviation from the
+   generic ones are documented.
+
+*/
+
+/* --- Manage the default encoding ---------------------------------------- */
+
+/* Returns a pointer to the default encoding (UTF-8) of the
+   Unicode object unicode and the size of the encoded representation
+   in bytes stored in *size.
+
+   In case of an error, no *size is set.
+
+   This function caches the UTF-8 encoded string in the unicodeobject
+   and subsequent calls will return the same string.  The memory is released
+   when the unicodeobject is deallocated.
+
+   _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to
+   support the previous internal function with the same behaviour.
+
+   *** This API is for interpreter INTERNAL USE ONLY and will likely
+   *** be removed or changed in the future.
+
+   *** If you need to access the Unicode object as UTF-8 bytes string,
+   *** please use PyUnicode_AsUTF8String() instead.
+*/
+
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(char *) PyUnicode_AsUTF8AndSize(
+    PyObject *unicode,
+    Py_ssize_t *size);
+#define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize
+#endif
+
+/* Returns a pointer to the default encoding (UTF-8) of the
+   Unicode object unicode.
+
+   Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation
+   in the unicodeobject.
+
+   _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to
+   support the previous internal function with the same behaviour.
+
+   Use of this API is DEPRECATED since no size information can be
+   extracted from the returned data.
+
+   *** This API is for interpreter INTERNAL USE ONLY and will likely
+   *** be removed or changed for Python 3.1.
+
+   *** If you need to access the Unicode object as UTF-8 bytes string,
+   *** please use PyUnicode_AsUTF8String() instead.
+
+*/
+
+#ifndef Py_LIMITED_API
 #define _PyUnicode_AsString PyUnicode_AsUTF8
-
-PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size);
+#endif
 
 Py_LOCAL_INLINE(size_t) Py_UNICODE_strlen(const Py_UNICODE *u)
 {
diff --git a/pypy/module/cpyext/parse/cpyext_unicodeobject.h b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
--- a/pypy/module/cpyext/parse/cpyext_unicodeobject.h
+++ b/pypy/module/cpyext/parse/cpyext_unicodeobject.h
@@ -1,13 +1,199 @@
+/* --- Internal Unicode Format -------------------------------------------- */
+
+
+/* Py_UNICODE was the native Unicode storage format (code unit) used by
+   Python and represents a single Unicode element in the Unicode type.
+   With PEP 393, Py_UNICODE is deprecated and replaced with a
+   typedef to wchar_t. */
+
+#define PY_UNICODE_TYPE wchar_t
+typedef wchar_t Py_UNICODE;
+
+/* Py_UCS4 and Py_UCS2 are typedefs for the respective
+   unicode representations. */
 typedef unsigned int Py_UCS4;
-/* On PyPy, Py_UNICODE is always wchar_t */
-#define PY_UNICODE_TYPE wchar_t
-typedef PY_UNICODE_TYPE Py_UNICODE;
+typedef unsigned short Py_UCS2;
+typedef unsigned char Py_UCS1;
 
-#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
+/* --- Unicode Type ------------------------------------------------------- */
 
 typedef struct {
+        /*
+           SSTATE_NOT_INTERNED (0)
+           SSTATE_INTERNED_MORTAL (1)
+           SSTATE_INTERNED_IMMORTAL (2)
+
+           If interned != SSTATE_NOT_INTERNED, the two references from the
+           dictionary to this object are *not* counted in ob_refcnt.
+         */
+        unsigned int interned;
+        /* Character size:
+
+           - PyUnicode_WCHAR_KIND (0):
+
+             * character type = wchar_t (16 or 32 bits, depending on the
+               platform)
+
+           - PyUnicode_1BYTE_KIND (1):
+
+             * character type = Py_UCS1 (8 bits, unsigned)
+             * all characters are in the range U+0000-U+00FF (latin1)
+             * if ascii is set, all characters are in the range U+0000-U+007F
+               (ASCII), otherwise at least one character is in the range
+               U+0080-U+00FF
+
+           - PyUnicode_2BYTE_KIND (2):
+
+             * character type = Py_UCS2 (16 bits, unsigned)
+             * all characters are in the range U+0000-U+FFFF (BMP)
+             * at least one character is in the range U+0100-U+FFFF
+
+           - PyUnicode_4BYTE_KIND (4):
+
+             * character type = Py_UCS4 (32 bits, unsigned)
+             * all characters are in the range U+0000-U+10FFFF
+             * at least one character is in the range U+10000-U+10FFFF
+         */
+        unsigned int kind;
+        /* Compact is with respect to the allocation scheme. Compact unicode
+           objects only require one memory block while non-compact objects use
+           one block for the PyUnicodeObject struct and another for its data
+           buffer. */
+        unsigned int compact;
+        /* The string only contains characters in the range U+0000-U+007F (ASCII)
+           and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
+           set, use the PyASCIIObject structure. */
+        unsigned int ascii;
+        /* The ready flag indicates whether the object layout is initialized
+           completely. This means that this is either a compact object, or
+           the data pointer is filled out. The bit is redundant, and helps
+           to minimize the test in PyUnicode_IS_READY(). */
+        unsigned int ready;
+        /* Padding to ensure that PyUnicode_DATA() is always aligned to
+           4 bytes (see issue #19537 on m68k). */
+        /* not on PyPy */
+    } _PyASCIIObject_state_t;
+
+/* ASCII-only strings created through PyUnicode_New use the PyASCIIObject
+   structure. state.ascii and state.compact are set, and the data
+   immediately follow the structure. utf8_length and wstr_length can be found
+   in the length field; the utf8 pointer is equal to the data pointer. */
+typedef struct {
+    /* There are 4 forms of Unicode strings:
+
+       - compact ascii:
+
+         * structure = PyASCIIObject
+         * test: PyUnicode_IS_COMPACT_ASCII(op)
+         * kind = PyUnicode_1BYTE_KIND
+         * compact = 1
+         * ascii = 1
+         * ready = 1
+         * (length is the length of the utf8 and wstr strings)
+         * (data starts just after the structure)
+         * (since ASCII is decoded from UTF-8, the utf8 string are the data)
+
+       - compact:
+
+         * structure = PyCompactUnicodeObject
+         * test: PyUnicode_IS_COMPACT(op) && !PyUnicode_IS_ASCII(op)
+         * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+           PyUnicode_4BYTE_KIND
+         * compact = 1
+         * ready = 1
+         * ascii = 0
+         * utf8 is not shared with data
+         * utf8_length = 0 if utf8 is NULL
+         * wstr is shared with data and wstr_length=length
+           if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+           or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_t)=4
+         * wstr_length = 0 if wstr is NULL
+         * (data starts just after the structure)
+
+       - legacy string, not ready:
+
+         * structure = PyUnicodeObject
+         * test: kind == PyUnicode_WCHAR_KIND
+         * length = 0 (use wstr_length)
+         * hash = -1
+         * kind = PyUnicode_WCHAR_KIND
+         * compact = 0
+         * ascii = 0
+         * ready = 0
+         * interned = SSTATE_NOT_INTERNED
+         * wstr is not NULL
+         * data.any is NULL
+         * utf8 is NULL
+         * utf8_length = 0
+
+       - legacy string, ready:
+
+         * structure = PyUnicodeObject structure
+         * test: !PyUnicode_IS_COMPACT(op) && kind != PyUnicode_WCHAR_KIND
+         * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+           PyUnicode_4BYTE_KIND
+         * compact = 0
+         * ready = 1
+         * data.any is not NULL
+         * utf8 is shared and utf8_length = length with data.any if ascii = 1
+         * utf8_length = 0 if utf8 is NULL
+         * wstr is shared with data.any and wstr_length = length
+           if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+           or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_4)=4
+         * wstr_length = 0 if wstr is NULL
+
+       Compact strings use only one memory block (structure + characters),
+       whereas legacy strings use one block for the structure and one block
+       for characters.
+
+       Legacy strings are created by PyUnicode_FromUnicode() and
+       PyUnicode_FromStringAndSize(NULL, size) functions. They become ready
+       when PyUnicode_READY() is called.
+
+       See also _PyUnicode_CheckConsistency().
+    */
     PyObject_HEAD
-    Py_UNICODE *buffer;
-    Py_ssize_t length;
-    char *utf8buffer;
+    Py_ssize_t length;          /* Number of code points in the string */
+    //Py_hash_t hash;             /* Hash value; -1 if not set */
+    _PyASCIIObject_state_t state;
+    wchar_t *wstr;              /* wchar_t representation (null-terminated) */
+} PyASCIIObject;
+
+/* Non-ASCII strings allocated through PyUnicode_New use the
+   PyCompactUnicodeObject structure. state.compact is set, and the data
+   immediately follow the structure. */
+typedef struct {
+    PyASCIIObject _base;
+    Py_ssize_t utf8_length;     /* Number of bytes in utf8, excluding the
+                                 * terminating \0. */
+    char *utf8;                 /* UTF-8 representation (null-terminated) */
+    Py_ssize_t wstr_length;     /* Number of code points in wstr, possible
+                                 * surrogates count as two code points. */
+} PyCompactUnicodeObject;
+
+/* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the
+   PyUnicodeObject structure. The actual string data is initially in the wstr
+   block, and copied into the data block using _PyUnicode_Ready. */
+typedef struct {
+    PyCompactUnicodeObject _base;
+    void* data;                     /* Canonical, smallest-form Unicode buffer */
 } PyUnicodeObject;
+
+
+/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */
+
+/* Values for PyASCIIObject.state: */
+
+/* Interning state. */
+#define SSTATE_NOT_INTERNED 0
+#define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+
+/* --- Constants ---------------------------------------------------------- */
+
+/* This Unicode character will be used as replacement character during
+   decoding if the errors argument is set to "replace". Note: the
+   Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
+   Unicode 3.0. */
+
+#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -471,7 +471,7 @@
                             readonly=widen(pybuf.c_readonly),
                             releasebufferproc = rbp)
         fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        return space.newbuffer(buf, itemsize=buf.itemsize)
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -79,6 +79,12 @@
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
 
+    def test_string_buf(self):
+        module = self.import_module(name='array')
+        arr = module.array('u', '123')
+        view = memoryview(arr)
+        assert view.itemsize == 4
+
     def test_subclass(self):
         import struct
         module = self.import_module(name='array')
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 import sys, py
 from pypy.module.cpyext.unicodeobject import *
+from pypy.module.cpyext.unicodeobject import _PyUnicode_Ready
 
 class AppTestUnicodeObject(AppTestCpythonExtensionBase):
     def test_unicodeobject(self):
@@ -28,7 +29,7 @@
                  }
 #ifdef PYPY_VERSION
                  // Slightly silly test that tp_basicsize is reasonable.
-                 if(s->ob_type->tp_basicsize != sizeof(void*)*6)
+                 if(s->ob_type->tp_basicsize != sizeof(void*)*12)
                      result = s->ob_type->tp_basicsize;
 #endif  // PYPY_VERSION
                  Py_DECREF(s);
@@ -40,7 +41,7 @@
                  int result = 0;
 
                  if(PyUnicode_GetLength(s) != 11) {
-                     result = -PyUnicode_GetSize(s);
+                     result = -PyUnicode_GetLength(s);
                  }
                  Py_DECREF(s);
                  return PyLong_FromLong(result);
@@ -212,10 +213,8 @@
 
 class TestUnicode(BaseApiTest):
     def test_unicodeobject(self, space):
-        assert PyUnicode_GET_SIZE(space, space.wrap(u'sp&#228;m')) == 4
         assert PyUnicode_GetSize(space, space.wrap(u'sp&#228;m')) == 4
         unichar = rffi.sizeof(Py_UNICODE)
-        assert PyUnicode_GET_DATA_SIZE(space, space.wrap(u'sp&#228;m')) == 4 * unichar
 
         encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, ))
         w_default_encoding = space.call_function(
@@ -225,13 +224,11 @@
 
     def test_AS(self, space):
         word = space.wrap(u'spam')
-        array = rffi.cast(rffi.CWCHARP, PyUnicode_AS_DATA(space, word))
-        array2 = PyUnicode_AS_UNICODE(space, word)
-        array3 = PyUnicode_AsUnicode(space, word)
+        array = rffi.cast(rffi.CWCHARP, PyUnicode_AsUnicode(space, word))
+        array2 = PyUnicode_AsUnicode(space, word)
         for (i, char) in enumerate(space.unwrap(word)):
             assert array[i] == char
             assert array2[i] == char
-            assert array3[i] == char
         with raises_w(space, TypeError):
             PyUnicode_AsUnicode(space, space.newbytes('spam'))
 
@@ -284,22 +281,23 @@
     def test_unicode_resize(self, space):
         py_uni = new_empty_unicode(space, 10)
         ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
-        py_uni.c_buffer[0] = u'a'
-        py_uni.c_buffer[1] = u'b'
-        py_uni.c_buffer[2] = u'c'
+        buf = get_wbuffer(py_uni)
+        buf[0] = u'a'
+        buf[1] = u'b'
+        buf[2] = u'c'
         ar[0] = rffi.cast(PyObject, py_uni)
         PyUnicode_Resize(space, ar, 3)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
-        assert py_uni.c_length == 3
-        assert py_uni.c_buffer[1] == u'b'
-        assert py_uni.c_buffer[3] == u'\x00'
+        assert get_wsize(py_uni) == 3
+        assert get_wbuffer(py_uni)[1] == u'b'
+        assert get_wbuffer(py_uni)[3] == u'\x00'
         # the same for growing
         ar[0] = rffi.cast(PyObject, py_uni)
         PyUnicode_Resize(space, ar, 10)
         py_uni = rffi.cast(PyUnicodeObject, ar[0])
-        assert py_uni.c_length == 10
-        assert py_uni.c_buffer[1] == 'b'
-        assert py_uni.c_buffer[10] == '\x00'
+        assert get_wsize(py_uni) == 10
+        assert get_wbuffer(py_uni)[1] == 'b'
+        assert get_wbuffer(py_uni)[10] == '\x00'
         Py_DecRef(space, ar[0])
         lltype.free(ar, flavor='raw')
 
@@ -623,13 +621,13 @@
         count1 = space.int_w(space.len(w_x))
         target_chunk = lltype.malloc(rffi.CWCHARP.TO, count1, flavor='raw')
 
-        x_chunk = PyUnicode_AS_UNICODE(space, w_x)
+        x_chunk = PyUnicode_AsUnicode(space, w_x)
         Py_UNICODE_COPY(space, target_chunk, x_chunk, 4)
         w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, 4))
 
         assert space.eq_w(w_y, space.wrap(u"abcd"))
 
-        size = PyUnicode_GET_SIZE(space, w_x)
+        size = PyUnicode_GetSize(space, w_x)
         Py_UNICODE_COPY(space, target_chunk, x_chunk, size)
         w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size))
 
@@ -758,3 +756,32 @@
                 PyUnicode_Splitlines(space, w_str, 0)))
         assert r"['a\n', 'b\n', 'c\n', 'd']" == space.unwrap(space.repr(
                 PyUnicode_Splitlines(space, w_str, 1)))
+
+    def test_Ready(self, space):
+        w_str = space.wrap(u'abc')  # ASCII
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 1
+        assert get_ascii(py_str) == 1
+
+        w_str = space.wrap(u'caf&#233;')  # latin1
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 1
+        assert get_ascii(py_str) == 0
+
+        w_str = space.wrap(u'&#1056;&#1086;&#1089;&#1089;&#1080;&#769;&#1103;')  # UCS2
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 2
+        assert get_ascii(py_str) == 0
+
+        w_str = space.wrap(u'***\U0001f4a9***')  # UCS4
+        py_str = as_pyobj(space, w_str)
+        assert get_kind(py_str) == 0
+        _PyUnicode_Ready(space, w_str)
+        assert get_kind(py_str) == 4
+        assert get_ascii(py_str) == 0
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -1,5 +1,8 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.runicode import unicode_encode_latin_1, unicode_encode_utf_16
+from rpython.rlib.rarithmetic import widen
+
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
@@ -8,7 +11,7 @@
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
-    make_typedescr, get_typedescr)
+    make_typedescr, get_typedescr, as_pyobj)
 from pypy.module.cpyext.bytesobject import PyBytes_Check, PyBytes_FromObject
 from pypy.module._codecs.interp_codecs import CodecState
 from pypy.objspace.std import unicodeobject
@@ -37,6 +40,12 @@
 
 PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
 
+MAX_UNICODE = 1114111
+WCHAR_KIND = 0
+_1BYTE_KIND = 1
+_2BYTE_KIND = 2
+_4BYTE_KIND = 4
+
 
 def new_empty_unicode(space, length):
     """
@@ -46,28 +55,26 @@
     """
     typedescr = get_typedescr(space.w_unicode.layout.typedef)
     py_obj = typedescr.allocate(space, space.w_unicode)
-    py_uni = rffi.cast(PyUnicodeObject, py_obj)
 
     buflen = length + 1
-    py_uni.c_length = length
-    py_uni.c_buffer = lltype.malloc(rffi.CWCHARP.TO, buflen,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
-    return py_uni
+    set_wsize(py_obj, length)
+    set_wbuffer(py_obj,
+        lltype.malloc(
+            rffi.CWCHARP.TO, buflen, flavor='raw', zero=True,
+            add_memory_pressure=True))
+    return py_obj
 
 def unicode_attach(space, py_obj, w_obj, w_userdata=None):
     "Fills a newly allocated PyUnicodeObject with a unicode string"
-    py_unicode = rffi.cast(PyUnicodeObject, py_obj)
-    py_unicode.c_length = len(space.unicode_w(w_obj))
-    py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO)
+    set_wsize(py_obj, len(space.unicode_w(w_obj)))
+    set_wbuffer(py_obj, lltype.nullptr(rffi.CWCHARP.TO))
 
 def unicode_realize(space, py_obj):
     """
     Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
     be modified after this call.
     """
-    py_uni = rffi.cast(PyUnicodeObject, py_obj)
-    s = rffi.wcharpsize2unicode(py_uni.c_buffer, py_uni.c_length)
+    s = rffi.wcharpsize2unicode(get_wbuffer(py_obj), get_wsize(py_obj))
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
     w_obj.__init__(s)
@@ -76,14 +83,81 @@
 
 @slot_function([PyObject], lltype.Void)
 def unicode_dealloc(space, py_obj):
-    py_unicode = rffi.cast(PyUnicodeObject, py_obj)
-    if py_unicode.c_buffer:
-        lltype.free(py_unicode.c_buffer, flavor="raw")
-    if py_unicode.c_utf8buffer:
-        lltype.free(py_unicode.c_utf8buffer, flavor="raw")
+    if get_wbuffer(py_obj):
+        lltype.free(get_wbuffer(py_obj), flavor="raw")
+    if get_utf8(py_obj):
+        lltype.free(get_utf8(py_obj), flavor="raw")
     from pypy.module.cpyext.object import _dealloc
     _dealloc(space, py_obj)
 
+def get_len(py_obj):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    return py_obj.c_length
+
+def set_len(py_obj, n):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    py_obj.c_length = n
+
+def get_state(py_obj):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    return py_obj.c_state
+
+def get_kind(py_obj):
+    return get_state(py_obj).c_kind
+
+def set_kind(py_obj, value):
+    get_state(py_obj).c_kind = cts.cast('unsigned int', value)
+
+def get_ascii(py_obj):
+    return get_state(py_obj).c_ascii
+
+def set_ascii(py_obj, value):
+    get_state(py_obj).c_ascii = cts.cast('unsigned int', value)
+
+def set_ready(py_obj, value):
+    get_state(py_obj).c_ready = cts.cast('unsigned int', value)
+
+def get_wbuffer(py_obj):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    return py_obj.c_wstr
+
+def set_wbuffer(py_obj, wbuf):
+    py_obj = cts.cast('PyASCIIObject*', py_obj)
+    py_obj.c_wstr = wbuf
+
+def get_utf8_len(py_obj):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    return py_obj.c_utf8_length
+
+def set_utf8_len(py_obj, n):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    py_obj.c_utf8_length = n
+
+def get_utf8(py_obj):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    return py_obj.c_utf8
+
+def set_utf8(py_obj, buf):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    py_obj.c_utf8 = buf
+
+def get_wsize(py_obj):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    return py_obj.c_wstr_length
+
+def set_wsize(py_obj, value):
+    py_obj = cts.cast('PyCompactUnicodeObject*', py_obj)
+    py_obj.c_wstr_length = value
+
+def get_data(py_obj):
+    py_obj = cts.cast('PyUnicodeObject*', py_obj)
+    return py_obj.c_data
+
+def set_data(py_obj, p_data):
+    py_obj = cts.cast('PyUnicodeObject*', py_obj)
+    py_obj.c_data = p_data
+
+
 @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL)
 def Py_UNICODE_ISSPACE(space, ch):
     """Return 1 or 0 depending on whether ch is a whitespace character."""
@@ -181,48 +255,52 @@
     """Get the maximum ordinal for a Unicode character."""
     return runicode.UNICHR(runicode.MAXUNICODE)
 
- at cpython_api([rffi.VOIDP], rffi.CCHARP, error=CANNOT_FAIL)
-def PyUnicode_AS_DATA(space, ref):
-    """Return a pointer to the internal buffer of the object. o has to be a
-    PyUnicodeObject (not checked)."""
-    return rffi.cast(rffi.CCHARP, PyUnicode_AS_UNICODE(space, ref))
-
- at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyUnicode_GET_DATA_SIZE(space, w_obj):
-    """Return the size of the object's internal buffer in bytes.  o has to be a
-    PyUnicodeObject (not checked)."""
-    return rffi.sizeof(Py_UNICODE) * PyUnicode_GET_SIZE(space, w_obj)
-
- at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PyUnicode_GET_SIZE(space, w_obj):
-    """Return the size of the object.  obj is a PyUnicodeObject (not
-    checked)."""
-    return space.len_w(w_obj)
-
- at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
-def PyUnicode_GET_LENGTH(space, w_obj):
-    """Return the length of the Unicode string, in code points.
-    o has to be a Unicode object in the "canonical" representation
-    (not checked)."""
+ at cts.decl("int _PyUnicode_Ready(PyObject *unicode)", error=-1)
+def _PyUnicode_Ready(space, w_obj):
     assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
-    return space.len_w(w_obj)
-
- at cpython_api([PyObject], rffi.INT, error=CANNOT_FAIL)
-def PyUnicode_IS_READY(space, w_obj):
-    # PyPy is always ready.
-    return space.w_True
-
- at cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL)
-def PyUnicode_AS_UNICODE(space, ref):
-    """Return a pointer to the internal Py_UNICODE buffer of the object.  ref
-    has to be a PyUnicodeObject (not checked)."""
-    ref_unicode = rffi.cast(PyUnicodeObject, ref)
-    if not ref_unicode.c_buffer:
-        # Copy unicode buffer
-        w_unicode = from_ref(space, rffi.cast(PyObject, ref))
-        u = space.unicode_w(w_unicode)
-        ref_unicode.c_buffer = rffi.unicode2wcharp(u)
-    return ref_unicode.c_buffer
+    py_obj = as_pyobj(space, w_obj)
+    assert widen(get_kind(py_obj)) == WCHAR_KIND
+    maxchar = 0
+    for c in w_obj._value:
+        if ord(c) > maxchar:
+            maxchar = ord(c)
+            if maxchar > MAX_UNICODE:
+                raise oefmt(space.w_ValueError,
+                    "Character U+%d is not in range [U+0000; U+10ffff]",
+                    maxchar)
+    if maxchar < 256:
+        ucs1_data = rffi.str2charp(unicode_encode_latin_1(
+            w_obj._value, len(w_obj._value), errors='strict'))
+        set_data(py_obj, cts.cast('void*', ucs1_data))
+        set_kind(py_obj, _1BYTE_KIND)
+        if maxchar < 128:
+            set_ascii(py_obj, 1)
+            set_utf8(py_obj, cts.cast('char*', get_data(py_obj)))
+            set_utf8_len(py_obj, get_wsize(py_obj))
+        else:
+            set_ascii(py_obj, 0)
+            set_utf8(py_obj, cts.cast('char *', 0))
+            set_utf8_len(py_obj, 0)
+    elif maxchar < 65536:
+        # XXX: assumes that sizeof(wchar_t) == 4
+        ucs2_str = unicode_encode_utf_16(
+            w_obj._value, len(w_obj._value), errors='strict')
+        ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
+        set_data(py_obj, cts.cast('void*', ucs2_data))
+        set_len(py_obj, get_wsize(py_obj))
+        set_kind(py_obj, _2BYTE_KIND)
+        set_utf8(py_obj, cts.cast('char *', 0))
+        set_utf8_len(py_obj, 0)
+    else:
+        # XXX: assumes that sizeof(wchar_t) == 4
+        ucs4_data = get_wbuffer(py_obj)
+        set_data(py_obj, cts.cast('void*', ucs4_data))
+        set_len(py_obj, get_wsize(py_obj))
+        set_kind(py_obj, _4BYTE_KIND)
+        set_utf8(py_obj, cts.cast('char *', 0))
+        set_utf8_len(py_obj, 0)
+    set_ready(py_obj, 1)
+    return 0
 
 @cpython_api([PyObject], rffi.CWCHARP)
 def PyUnicode_AsUnicode(space, ref):
@@ -232,19 +310,23 @@
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
     if not space.issubtype_w(w_type, space.w_unicode):
         raise oefmt(space.w_TypeError, "expected unicode object")
-    return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
+    if not get_wbuffer(ref):
+        # Copy unicode buffer
+        w_unicode = from_ref(space, rffi.cast(PyObject, ref))
+        u = space.unicode_w(w_unicode)
+        set_wbuffer(ref, rffi.unicode2wcharp(u))
+    return get_wbuffer(ref)
 
 @cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)")
 def PyUnicode_AsUTF8(space, ref):
-    ref_unicode = rffi.cast(PyUnicodeObject, ref)
-    if not ref_unicode.c_utf8buffer:
+    if not get_utf8(ref):
         # Copy unicode buffer
         w_unicode = from_ref(space, ref)
         w_encoded = unicodeobject.encode_object(space, w_unicode, "utf-8",
                                                 "strict")
         s = space.bytes_w(w_encoded)
-        ref_unicode.c_utf8buffer = rffi.str2charp(s)
-    return ref_unicode.c_utf8buffer
+        set_utf8(ref, rffi.str2charp(s))
+    return get_utf8(ref)
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
 def PyUnicode_GetSize(space, ref):
@@ -254,20 +336,19 @@
     Please migrate to using PyUnicode_GetLength().
     """
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_unicode:
-        ref = rffi.cast(PyUnicodeObject, ref)
-        return ref.c_length
+        return get_wsize(ref)
     else:
         w_obj = from_ref(space, ref)
         return space.len_w(w_obj)
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
-def PyUnicode_GetLength(space, w_unicode):
+def PyUnicode_GetLength(space, unicode):
     """Return the length of the Unicode object, in code points."""
     # XXX: this is a stub
-    if not PyUnicode_Check(space, w_unicode):
+    if not PyUnicode_Check(space, unicode):
         PyErr_BadArgument(space)
     #PyUnicode_READY(w_unicode)
-    return PyUnicode_GET_LENGTH(space, w_unicode)
+    return get_len(unicode)
 
 @cpython_api([PyObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_AsWideChar(space, ref, buf, size):
@@ -278,9 +359,8 @@
     string may or may not be 0-terminated.  It is the responsibility of the caller
     to make sure that the wchar_t string is 0-terminated in case this is
     required by the application."""
-    ref = rffi.cast(PyUnicodeObject, ref)
-    c_buffer = PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref))
-    c_length = ref.c_length
+    c_buffer = PyUnicode_AsUnicode(space, ref)
+    c_length = get_wsize(ref)
 
     # If possible, try to copy the 0-termination as well
     if size > c_length:
@@ -359,7 +439,7 @@
         s = rffi.wcharpsize2unicode(wchar_p, length)
         return make_ref(space, space.wrap(s))
     else:
-        return rffi.cast(PyObject, new_empty_unicode(space, length))
+        return new_empty_unicode(space, length)
 
 @cpython_api([CONST_WSTRING, Py_ssize_t], PyObject, result_is_ll=True)
 def PyUnicode_FromWideChar(space, wchar_p, length):
@@ -550,7 +630,7 @@
         return make_ref(space, PyUnicode_DecodeUTF8(
             space, s, size, lltype.nullptr(rffi.CCHARP.TO)))
     else:
-        return rffi.cast(PyObject, new_empty_unicode(space, size))
+        return new_empty_unicode(space, size)
 
 @cpython_api([rffi.INT_real], PyObject)
 def PyUnicode_FromOrdinal(space, ordinal):
@@ -565,8 +645,8 @@
 @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1)
 def PyUnicode_Resize(space, ref, newsize):
     # XXX always create a new string so far
-    py_uni = rffi.cast(PyUnicodeObject, ref[0])
-    if not py_uni.c_buffer:
+    py_obj = ref[0]
+    if not get_wbuffer(py_obj):
         raise oefmt(space.w_SystemError,
                     "PyUnicode_Resize called on already created string")
     try:
@@ -576,11 +656,11 @@
         ref[0] = lltype.nullptr(PyObject.TO)
         raise
     to_cp = newsize
-    oldsize = py_uni.c_length
+    oldsize = get_wsize(py_obj)
     if oldsize < newsize:
         to_cp = oldsize
     for i in range(to_cp):
-        py_newuni.c_buffer[i] = py_uni.c_buffer[i]
+        get_wbuffer(py_newuni)[i] = get_wbuffer(py_obj)[i]
     Py_DecRef(space, ref[0])
     ref[0] = rffi.cast(PyObject, py_newuni)
     return 0
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -196,7 +196,7 @@
     def newseqiter(self, x):
         return w_some_obj()
 
-    def newbuffer(self, x):
+    def newbuffer(self, x, itemsize=1):
         return w_some_obj()
 
     def marshal_w(self, w_obj):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -295,7 +295,7 @@
         return W_LongObject.fromint(self, val)
 
     @specialize.argtype(1)
-    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type
         return W_LongObject.fromrarith_int(val)
 
     def newlong_from_rbigint(self, val):
@@ -350,8 +350,8 @@
     def newseqiter(self, w_obj):
         return W_SeqIterObject(w_obj)
 
-    def newbuffer(self, w_obj):
-        return W_MemoryView(w_obj)
+    def newbuffer(self, w_obj, itemsize=1):
+        return W_MemoryView(w_obj, itemsize=itemsize)
 
     def newbytes(self, s):
         return W_BytesObject(s)

From pypy.commits at gmail.com  Sat Feb  4 13:37:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 10:37:26 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: A branch to implement the
 move_to_end(last=True/False) on RPython
Message-ID: <58961f66.828bdf0a.8e041.d129@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89924:5a406ad32b05
Date: 2017-02-04 19:33 +0100
http://bitbucket.org/pypy/pypy/changeset/5a406ad32b05/

Log:	A branch to implement the move_to_end(last=True/False) on RPython
	ordered dicts


From pypy.commits at gmail.com  Sat Feb  4 13:37:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 10:37:28 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: First,
 a commit which should make not much measurable difference: remove
Message-ID: <58961f68.0a8edf0a.8962.c6b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89925:9a5973e899f5
Date: 2017-02-04 19:36 +0100
http://bitbucket.org/pypy/pypy/changeset/9a5973e899f5/

Log:	First, a commit which should make not much measurable difference:
	remove FLAG_DELETE, replace it with a call to
	ll_call_delete_by_entry_index()

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -543,6 +543,8 @@
         assert False
 
 def ll_call_delete_by_entry_index(d, hash, i):
+    # only called from _ll_dict_del, whose @jit.look_inside_iff
+    # condition should control when we get inside here with the jit
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
@@ -805,13 +807,14 @@
     ll_dict_delitem_with_hash(d, key, d.keyhash(key))
 
 def ll_dict_delitem_with_hash(d, key, hash):
-    index = d.lookup_function(d, key, hash, FLAG_DELETE)
+    index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
-    _ll_dict_del(d, index)
+    _ll_dict_del(d, hash, index)
 
- at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, index):
+ at jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, hash, index):
+    ll_call_delete_by_entry_index(d, hash, index)
     d.entries.mark_deleted(index)
     d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
@@ -963,7 +966,6 @@
 
 FLAG_LOOKUP = 0
 FLAG_STORE = 1
-FLAG_DELETE = 2
 
 @specialize.memo()
 def _ll_ptr_to_array_of(T):
@@ -985,8 +987,6 @@
     if index >= VALID_OFFSET:
         checkingkey = entries[index - VALID_OFFSET].key
         if direct_compare and checkingkey == key:
-            if store_flag == FLAG_DELETE:
-                indexes[i] = rffi.cast(T, DELETED)
             return index - VALID_OFFSET   # found the entry
         if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
             # correct hash, maybe the key is e.g. a different pointer to
@@ -1000,8 +1000,6 @@
                     # the compare did major nasty stuff to the dict: start over
                     return ll_dict_lookup(d, key, hash, store_flag, T)
             if found:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET
         deletedslot = -1
     elif index == DELETED:
@@ -1030,8 +1028,6 @@
         elif index >= VALID_OFFSET:
             checkingkey = entries[index - VALID_OFFSET].key
             if direct_compare and checkingkey == key:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET   # found the entry
             if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
                 # correct hash, maybe the key is e.g. a different pointer to
@@ -1044,8 +1040,6 @@
                         # the compare did major nasty stuff to the dict: start over
                         return ll_dict_lookup(d, key, hash, store_flag, T)
                 if found:
-                    if store_flag == FLAG_DELETE:
-                        indexes[i] = rffi.cast(T, DELETED)
                     return index - VALID_OFFSET
         elif deletedslot == -1:
             deletedslot = intmask(i)
@@ -1066,6 +1060,10 @@
         perturb >>= PERTURB_SHIFT
     indexes[i] = rffi.cast(T, index + VALID_OFFSET)
 
+# the following function is only called from _ll_dict_del, whose
+# @jit.look_inside_iff condition should control when we get inside
+# here with the jit
+ at jit.unroll_safe
 def ll_dict_delete_by_entry_index(d, hash, locate_index, T):
     # Another simplified version of ll_dict_lookup() which locates a
     # hashtable entry with the given 'index' stored in it, and deletes it.
@@ -1390,8 +1388,6 @@
             break
         dic.num_ever_used_items -= 1
 
-    # we must remove the precise entry in the hashtable that points to 'i'
-    ll_call_delete_by_entry_index(dic, entries.hash(i), i)
     return i
 
 def ll_dict_popitem(ELEM, dic):
@@ -1400,21 +1396,23 @@
     r = lltype.malloc(ELEM.TO)
     r.item0 = recast(ELEM.TO.item0, entry.key)
     r.item1 = recast(ELEM.TO.item1, entry.value)
-    _ll_dict_del(dic, i)
+    _ll_dict_del(dic, dic.entries.hash(i), i)
     return r
 
 def ll_dict_pop(dic, key):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value
 
 def ll_dict_pop_default(dic, key, dfl):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         return dfl
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value

From pypy.commits at gmail.com  Sat Feb  4 14:17:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 11:17:41 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: Annotation,
 and the case of last=True
Message-ID: <589628d5.6787df0a.b4168.e2a4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89926:8286c209d677
Date: 2017-02-04 20:17 +0100
http://bitbucket.org/pypy/pypy/changeset/8286c209d677/

Log:	Annotation, and the case of last=True

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -14,7 +14,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
     SomeFloat, SomeIterator, SomePBC, SomeNone, SomeTypeOf, s_ImpossibleValue,
     s_Bool, s_None, s_Int, unionof, add_knowntypedata,
-    SomeWeakRef, SomeUnicodeString, SomeByteArray)
+    SomeWeakRef, SomeUnicodeString, SomeByteArray, SomeOrderedDict)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
 from rpython.annotator.binaryop import _dict_can_only_throw_keyerror
@@ -575,6 +575,13 @@
         pair(self, s_key).delitem()
     method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
 
+class __extend__(SomeOrderedDict):
+
+    def method_move_to_end(self, s_key, s_last):
+        assert s_Bool.contains(s_last)
+        pair(self, s_key).delitem()
+    method_move_to_end.can_only_throw = _dict_can_only_throw_keyerror
+
 @op.contains.register(SomeString)
 @op.contains.register(SomeUnicodeString)
 def contains_String(annotator, string, char):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,24 @@
         return
     d.delitem_with_hash(key, h)
 
+def _untranslated_move_to_end(d, key, last):
+    "NOT_RPYTHON"
+    value = d.pop(key)
+    if last:
+        d[key] = value
+    else:
+        items = d.items()
+        d.clear()
+        d[key] = value
+        d.update(items)
+
+ at specialize.call_location()
+def move_to_end(d, key, last=True):
+    if not we_are_translated():
+        _untranslated_move_to_end(d, key, last)
+        return
+    d.move_to_end(key, last)
+
 # ____________________________________________________________
 
 def import_from_mixin(M, special_methods=['__init__', '__del__']):
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config, try_inline)
+    fetch_translated_config, try_inline, move_to_end)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -679,6 +679,16 @@
     assert f(29) == 0
     interpret(f, [27])
 
+def test_rordereddict_move_to_end():
+    d = OrderedDict()
+    d['key1'] = 'val1'
+    d['key2'] = 'val2'
+    d['key3'] = 'val3'
+    move_to_end(d, 'key1')
+    assert d.items() == [('key2', 'val2'), ('key3', 'val3'), ('key1', 'val1')]
+    move_to_end(d, 'key1', last=False)
+    assert d.items() == [('key1', 'val1'), ('key2', 'val2'), ('key3', 'val3')]
+
 def test_import_from_mixin():
     class M:    # old-style
         def f(self):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,15 @@
         hop.exception_is_here()
         hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
 
+    def rtype_method_move_to_end(self, hop):
+        v_dict, v_key, v_last = hop.inputargs(
+            self, self.key_repr, lltype.Bool)
+        if not self.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)  # record that we know about it
+        hop.exception_is_here()
+        hop.gendirectcall(ll_dict_move_to_end, v_dict, v_key, v_last)
+
+
 class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
 
     def rtype_getitem((r_dict, r_key), hop):
@@ -542,18 +551,18 @@
         ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
         assert False
 
-def ll_call_delete_by_entry_index(d, hash, i):
+def ll_call_delete_by_entry_index(d, hash, i, replace_with):
     # only called from _ll_dict_del, whose @jit.look_inside_iff
     # condition should control when we get inside here with the jit
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_BYTE)
     elif fun == FUNC_SHORT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_SHORT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_SHORT)
     elif IS_64BIT and fun == FUNC_INT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_INT)
     elif fun == FUNC_LONG:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_LONG)
     else:
         # can't be still FUNC_MUST_REINDEX here
         ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
@@ -814,7 +823,7 @@
 
 @jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
 def _ll_dict_del(d, hash, index):
-    ll_call_delete_by_entry_index(d, hash, index)
+    ll_call_delete_by_entry_index(d, hash, index, DELETED)
     d.entries.mark_deleted(index)
     d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
@@ -1064,7 +1073,7 @@
 # @jit.look_inside_iff condition should control when we get inside
 # here with the jit
 @jit.unroll_safe
-def ll_dict_delete_by_entry_index(d, hash, locate_index, T):
+def ll_dict_delete_by_entry_index(d, hash, locate_index, replace_with, T):
     # Another simplified version of ll_dict_lookup() which locates a
     # hashtable entry with the given 'index' stored in it, and deletes it.
     # This *should* be safe against evil user-level __eq__/__hash__
@@ -1081,7 +1090,7 @@
         i = (i << 2) + i + perturb + 1
         i = i & mask
         perturb >>= PERTURB_SHIFT
-    indexes[i] = rffi.cast(T, DELETED)
+    indexes[i] = rffi.cast(T, replace_with)
 
 # ____________________________________________________________
 #
@@ -1416,3 +1425,35 @@
     value = dic.entries[index].value
     _ll_dict_del(dic, hash, index)
     return value
+
+def ll_dict_move_to_end(d, key, last):
+    assert last #XXX
+
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index < 0:
+        raise KeyError
+
+    if old_index == d.num_ever_used_items - 1:
+        return
+
+    # remove the entry at the old position
+    ENTRIES = lltype.typeOf(d.entries).TO
+    ENTRY = ENTRIES.OF
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    d.entries.mark_deleted(old_index)
+    if ENTRIES.must_clear_key:
+        old_entry.key = lltype.nullptr(ENTRY.key.TO)
+    if ENTRIES.must_clear_value:
+        old_entry.value = lltype.nullptr(ENTRY.value.TO)
+
+    # note a corner case: it is possible that 'replace_with' is just too
+    # large to fit in the type T used so far for the index.  But in that
+    # case, the list 'd.entries' is full, and the following call to
+    # _ll_dict_setitem_lookup_done() will necessarily reindex the dict.
+    # So in that case, this value of 'replace_with' should be ignored.
+    ll_call_delete_by_entry_index(d, hash, old_index,
+            replace_with = VALID_OFFSET + d.num_ever_used_items)
+    _ll_dict_setitem_lookup_done(d, key, value, hash, -1)
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -1,7 +1,8 @@
 import py
+import random
 from collections import OrderedDict
 
-from hypothesis import settings
+from hypothesis import settings, given, strategies
 from hypothesis.stateful import run_state_machine_as_test
 
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -145,14 +146,18 @@
         ll_d = rordereddict.ll_newdict(DICT)
         rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
         rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
-        ITER = rordereddict.get_ll_dictiter(lltype.Ptr(DICT))
+        assert [hlstr(entry.key) for entry in self._ll_iter(ll_d)] == ["k", "j"]
+
+    def _ll_iter(self, ll_d):
+        ITER = rordereddict.get_ll_dictiter(lltype.typeOf(ll_d))
         ll_iter = rordereddict.ll_dictiter(ITER, ll_d)
         ll_dictnext = rordereddict._ll_dictnext
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "k"
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "j"
-        py.test.raises(StopIteration, ll_dictnext, ll_iter)
+        while True:
+            try:
+                num = ll_dictnext(ll_iter)
+            except StopIteration:
+                break
+            yield ll_d.entries[num]
 
     def test_popitem(self):
         DICT = self._get_str_dict()
@@ -337,6 +342,31 @@
             num_nonfrees += (got > 0)
         assert d.resize_counter <= idx.getlength() * 2 - num_nonfrees * 3
 
+    @given(strategies.lists(strategies.integers(min_value=1, max_value=5)))
+    def test_direct_move_to_end(self, lst):
+        DICT = self._get_int_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, 1, 11)
+        rordereddict.ll_dict_setitem(ll_d, 2, 22)
+        def content():
+            return [(entry.key, entry.value) for entry in self._ll_iter(ll_d)]
+        for case in lst:
+            if case == 1:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, True)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 2:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, True)
+                assert content() == [(1, 11), (2, 22)]
+            elif case == 3:
+                py.test.raises(KeyError, rordereddict.ll_dict_move_to_end,
+                                                 ll_d, 3, True)
+            #elif case == 4:
+            #    rordereddict.ll_dict_move_to_end(ll_d, 2, False)
+            #    assert content() == [(2, 22), (1, 11)]
+            #elif case == 5:
+            #    rordereddict.ll_dict_move_to_end(ll_d, 1, False)
+            #    assert content() == [(1, 11), (2, 22)]
+
 
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
@@ -369,6 +399,32 @@
         res = self.interpret(func, [5])
         assert res == 6
 
+    def test_move_to_end(self):
+        def func():
+            d1 = OrderedDict()
+            d1['key1'] = 'value1'
+            d1['key2'] = 'value2'
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key1')
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key2')
+                assert d1.keys() == ['key1', 'key2']
+        func()
+        self.interpret(func, [])
+
+    def test_move_to_beginning(self):
+        def func():
+            d1 = OrderedDict()
+            d1['key1'] = 'value1'
+            d1['key2'] = 'value2'
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key2', last=False)
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key1', last=False)
+                assert d1.keys() == ['key1', 'key2']
+        func()
+        self.interpret(func, [])
+
 
 class ODictSpace(MappingSpace):
     MappingRepr = rodct.OrderedDictRepr

From pypy.commits at gmail.com  Sat Feb  4 15:46:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 12:46:36 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: fix
Message-ID: <58963dac.f0a6df0a.718f1.1c4c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89927:27ec95403b77
Date: 2017-02-04 20:26 +0100
http://bitbucket.org/pypy/pypy/changeset/27ec95403b77/

Log:	fix

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -1444,6 +1444,7 @@
     key = old_entry.key
     value = old_entry.value
     d.entries.mark_deleted(old_index)
+    d.num_live_items -= 1
     if ENTRIES.must_clear_key:
         old_entry.key = lltype.nullptr(ENTRY.key.TO)
     if ENTRIES.must_clear_value:

From pypy.commits at gmail.com  Sat Feb  4 15:46:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 12:46:38 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: move to beginning
Message-ID: <58963dae.2da9df0a.7f654.fff1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89928:6d1fd75ac17f
Date: 2017-02-04 21:46 +0100
http://bitbucket.org/pypy/pypy/changeset/6d1fd75ac17f/

Log:	move to beginning

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -821,9 +821,7 @@
         raise KeyError
     _ll_dict_del(d, hash, index)
 
- at jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, hash, index):
-    ll_call_delete_by_entry_index(d, hash, index, DELETED)
+def _ll_dict_del_entry(d, index):
     d.entries.mark_deleted(index)
     d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
@@ -835,6 +833,11 @@
     if ENTRIES.must_clear_value:
         entry.value = lltype.nullptr(ENTRY.value.TO)
 
+ at jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, hash, index):
+    ll_call_delete_by_entry_index(d, hash, index, DELETED)
+    _ll_dict_del_entry(d, index)
+
     if d.num_live_items == 0:
         # Dict is now empty.  Reset these fields.
         d.num_ever_used_items = 0
@@ -1427,8 +1430,12 @@
     return value
 
 def ll_dict_move_to_end(d, key, last):
-    assert last #XXX
+    if last:
+        ll_dict_move_to_end_really(d, key)
+    else:
+        ll_dict_move_to_beginning(d, key)
 
+def ll_dict_move_to_end_really(d, key):
     hash = d.keyhash(key)
     old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
     if old_index < 0:
@@ -1438,17 +1445,10 @@
         return
 
     # remove the entry at the old position
-    ENTRIES = lltype.typeOf(d.entries).TO
-    ENTRY = ENTRIES.OF
     old_entry = d.entries[old_index]
     key = old_entry.key
     value = old_entry.value
-    d.entries.mark_deleted(old_index)
-    d.num_live_items -= 1
-    if ENTRIES.must_clear_key:
-        old_entry.key = lltype.nullptr(ENTRY.key.TO)
-    if ENTRIES.must_clear_value:
-        old_entry.value = lltype.nullptr(ENTRY.value.TO)
+    _ll_dict_del_entry(d, old_index)
 
     # note a corner case: it is possible that 'replace_with' is just too
     # large to fit in the type T used so far for the index.  But in that
@@ -1458,3 +1458,89 @@
     ll_call_delete_by_entry_index(d, hash, old_index,
             replace_with = VALID_OFFSET + d.num_ever_used_items)
     _ll_dict_setitem_lookup_done(d, key, value, hash, -1)
+
+def ll_dict_move_to_beginning(d, key):
+    # In this function, we might do a bit more than the strict minimum
+    # of walks over parts of the array, trying to keep the code at least
+    # semi-reasonable, while the goal is still amortized constant-time
+    # over many calls.
+
+    # Call ll_dict_remove_deleted_items() first if there are too many
+    # deleted items.  Not a perfect solution, because lookup_function()
+    # might do random things with the dict and create many new deleted
+    # items.  Still, should be fine, because nothing crucially depends
+    # on this: the goal is to avoid the dictionary's list growing
+    # forever.
+    if d.num_live_items < len(d.entries) // 2 - 16:
+        ll_dict_remove_deleted_items(d)
+
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index <= 0:
+        if old_index < 0:
+            raise KeyError
+        else:
+            return
+
+    # the goal of the following is to set 'idst' to the number of
+    # deleted entries at the beginning, ensuring 'idst > 0'
+    must_reindex = False
+    if d.entries.valid(0):
+        # the first entry is valid, so we need to make room before.
+        new_allocated = _overallocate_entries_len(d.num_ever_used_items)
+        idst = ((new_allocated - d.num_ever_used_items) * 3) // 4
+        ll_assert(idst > 0, "overallocate did not do enough")
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+        rgc.ll_arraycopy(d.entries, newitems, 0, idst, d.num_ever_used_items)
+        d.entries = newitems
+        i = 0
+        while i < idst:
+            d.entries.mark_deleted(i)
+            i += 1
+        d.num_ever_used_items += idst
+        old_index += idst
+        must_reindex = True
+        idst -= 1
+    else:
+        idst = d.lookup_function_no >> FUNC_SHIFT
+        # All entries in range(0, idst) are deleted.  Check if more are
+        while not d.entries.valid(idst):
+            idst += 1
+        if idst == old_index:
+            d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                    (old_index << FUNC_SHIFT))
+            return
+        idst -= 1
+        d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                (idst << FUNC_SHIFT))
+
+    # remove the entry at the old position
+    ll_assert(d.entries.valid(old_index),
+              "ll_dict_move_to_beginning: lost old_index")
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    if hasattr(ENTRY, 'f_hash'):
+        ll_assert(old_entry.f_hash == hash,
+                  "ll_dict_move_to_beginning: bad hash")
+    _ll_dict_del_entry(d, old_index)
+
+    # put the entry at its new position
+    ll_assert(not d.entries.valid(idst),
+              "ll_dict_move_to_beginning: overwriting idst")
+    new_entry = d.entries[idst]
+    new_entry.key = key
+    new_entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        new_entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        new_entry.f_valid = True
+    d.num_live_items += 1
+
+    # fix the index
+    if must_reindex:
+        ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+    else:
+        ll_call_delete_by_entry_index(d, hash, old_index,
+                replace_with = VALID_OFFSET + idst)
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -360,12 +360,12 @@
             elif case == 3:
                 py.test.raises(KeyError, rordereddict.ll_dict_move_to_end,
                                                  ll_d, 3, True)
-            #elif case == 4:
-            #    rordereddict.ll_dict_move_to_end(ll_d, 2, False)
-            #    assert content() == [(2, 22), (1, 11)]
-            #elif case == 5:
-            #    rordereddict.ll_dict_move_to_end(ll_d, 1, False)
-            #    assert content() == [(1, 11), (2, 22)]
+            elif case == 4:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, False)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 5:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, False)
+                assert content() == [(1, 11), (2, 22)]
 
 
 class TestRDictDirectDummyKey(TestRDictDirect):
@@ -409,14 +409,6 @@
                 assert d1.keys() == ['key2', 'key1']
                 objectmodel.move_to_end(d1, 'key2')
                 assert d1.keys() == ['key1', 'key2']
-        func()
-        self.interpret(func, [])
-
-    def test_move_to_beginning(self):
-        def func():
-            d1 = OrderedDict()
-            d1['key1'] = 'value1'
-            d1['key2'] = 'value2'
             for i in range(20):
                 objectmodel.move_to_end(d1, 'key2', last=False)
                 assert d1.keys() == ['key2', 'key1']

From pypy.commits at gmail.com  Sat Feb  4 16:42:10 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 04 Feb 2017 13:42:10 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: A branch to refactor buffers and
 fix related cpyext issues
Message-ID: <58964ab2.4d821c0a.a8a2c.7b1d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r89929:ef18b16b4e97
Date: 2017-02-04 21:40 +0000
http://bitbucket.org/pypy/pypy/changeset/ef18b16b4e97/

Log:	A branch to refactor buffers and fix related cpyext issues


From pypy.commits at gmail.com  Sat Feb  4 16:42:12 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 04 Feb 2017 13:42:12 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: A failing test about itemsize
 calculation
Message-ID: <58964ab4.50a4df0a.2bbd1.28f4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r89930:10365a306083
Date: 2017-02-04 19:59 +0000
http://bitbucket.org/pypy/pypy/changeset/10365a306083/

Log:	A failing test about itemsize calculation

diff --git a/pypy/module/cpyext/test/test_buffer.py b/pypy/module/cpyext/test/test_buffer.py
--- a/pypy/module/cpyext/test/test_buffer.py
+++ b/pypy/module/cpyext/test/test_buffer.py
@@ -1,5 +1,23 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+class AppTestBuffer(AppTestCpythonExtensionBase):
+    def test_AsWriteBuffer(self):
+        import array
+        module = self.import_extension('buffer', [
+            ('write_buffer_len', 'METH_O',
+             """
+             void* buf;
+             Py_ssize_t buf_len;
+             if (PyObject_AsWriteBuffer(args, &buf, &buf_len) < 0) {
+                PyErr_SetString(PyExc_ValueError, "bad value");
+                return NULL;
+             }
+             return PyLong_FromLong(buf_len);
+             """)])
+        assert module.write_buffer_len(bytearray(b'123')) == 3
+        assert module.write_buffer_len(array.array('i', [1, 2, 3])) == 12
+
+
 class AppTestMmap(AppTestCpythonExtensionBase):
     def test_mmap_buffer(self):
         module = self.import_extension('mmap_buffer', [

From pypy.commits at gmail.com  Sat Feb  4 17:53:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 14:53:33 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: Add move_to_xx to the main
 hypothesis test. Simplify ll_dict_copy()
Message-ID: <58965b6d.c3161c0a.db8fc.8a4d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89931:481903aaf279
Date: 2017-02-04 22:03 +0100
http://bitbucket.org/pypy/pypy/changeset/481903aaf279/

Log:	Add move_to_xx to the main hypothesis test. Simplify ll_dict_copy()

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -1263,18 +1263,8 @@
     if hasattr(DICT, 'fnkeyhash'):
         newdict.fnkeyhash = dict.fnkeyhash
 
-    i = 0
-    while i < newdict.num_ever_used_items:
-        d_entry = newdict.entries[i]
-        entry = dict.entries[i]
-        ENTRY = lltype.typeOf(newdict.entries).TO.OF
-        d_entry.key = entry.key
-        if hasattr(ENTRY, 'f_valid'):
-            d_entry.f_valid = entry.f_valid
-        d_entry.value = entry.value
-        if hasattr(ENTRY, 'f_hash'):
-            d_entry.f_hash = entry.f_hash
-        i += 1
+    rgc.ll_arraycopy(dict.entries, newdict.entries, 0, 0,
+                     newdict.num_ever_used_items)
 
     ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
     return newdict
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1188,6 +1188,12 @@
         assert not self.ll_contains(self.l_dict, ll_key)
         self.removed_keys.append(key)
 
+    def move_to_end(self, key, last=True):
+        "For test_rordereddict"
+
+    def move_to_beginning(self, key):
+        self.move_to_end(key, last=False)
+
     def copydict(self):
         self.l_dict = self.ll_copy(self.l_dict)
         assert self.ll_len(self.l_dict) == len(self.reference)
@@ -1250,6 +1256,15 @@
         return builds(Action,
             just('delitem'), tuples(sampled_from(self.space.reference)))
 
+    def st_move_to_end(self):
+        return builds(Action,
+            just('move_to_end'), tuples(sampled_from(self.space.reference)))
+
+    def st_move_to_beginning(self):
+        return builds(Action,
+            just('move_to_beginning'),
+                tuples(sampled_from(self.space.reference)))
+
     def steps(self):
         if not self.space:
             return builds(Action, just('setup'), tuples(st_keys, st_values))
@@ -1258,7 +1273,8 @@
         if self.space.reference:
             return (
                 self.st_setitem() | sampled_from(global_actions) |
-                self.st_updateitem() | self.st_delitem())
+                self.st_updateitem() | self.st_delitem() |
+                self.st_move_to_end() | self.st_move_to_beginning())
         else:
             return (self.st_setitem() | sampled_from(global_actions))
 
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -421,6 +421,7 @@
 class ODictSpace(MappingSpace):
     MappingRepr = rodct.OrderedDictRepr
     new_reference = OrderedDict
+    moved_around = False
     ll_getitem = staticmethod(rodct.ll_dict_getitem)
     ll_setitem = staticmethod(rodct.ll_dict_setitem)
     ll_delitem = staticmethod(rodct.ll_dict_delitem)
@@ -465,9 +466,23 @@
     def removeindex(self):
         # remove the index, as done during translation for prebuilt dicts
         # (but cannot be done if we already removed a key)
-        if not self.removed_keys:
+        if not self.removed_keys and not self.moved_around:
             rodct.ll_no_initial_index(self.l_dict)
 
+    def move_to_end(self, key, last=True):
+        ll_key = self.ll_key(key)
+        rodct.ll_dict_move_to_end(self.l_dict, ll_key, last)
+        value = self.reference.pop(key)
+        if last:
+            self.reference[key] = value
+        else:
+            items = self.reference.items()
+            self.reference.clear()
+            self.reference[key] = value
+            self.reference.update(items)
+        # prevent ll_no_initial_index()
+        self.moved_around = True
+
     def fullcheck(self):
         # overridden to also check key order
         assert self.ll_len(self.l_dict) == len(self.reference)

From pypy.commits at gmail.com  Sun Feb  5 02:10:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 23:10:45 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: rename beginning/really_end =>
 first/last
Message-ID: <5896cff5.a1aedf0a.ff632.96c0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89932:6670eee6f62f
Date: 2017-02-05 08:10 +0100
http://bitbucket.org/pypy/pypy/changeset/6670eee6f62f/

Log:	rename beginning/really_end => first/last

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -1421,11 +1421,11 @@
 
 def ll_dict_move_to_end(d, key, last):
     if last:
-        ll_dict_move_to_end_really(d, key)
+        ll_dict_move_to_last(d, key)
     else:
-        ll_dict_move_to_beginning(d, key)
+        ll_dict_move_to_first(d, key)
 
-def ll_dict_move_to_end_really(d, key):
+def ll_dict_move_to_last(d, key):
     hash = d.keyhash(key)
     old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
     if old_index < 0:
@@ -1449,7 +1449,7 @@
             replace_with = VALID_OFFSET + d.num_ever_used_items)
     _ll_dict_setitem_lookup_done(d, key, value, hash, -1)
 
-def ll_dict_move_to_beginning(d, key):
+def ll_dict_move_to_first(d, key):
     # In this function, we might do a bit more than the strict minimum
     # of walks over parts of the array, trying to keep the code at least
     # semi-reasonable, while the goal is still amortized constant-time
@@ -1506,19 +1506,19 @@
 
     # remove the entry at the old position
     ll_assert(d.entries.valid(old_index),
-              "ll_dict_move_to_beginning: lost old_index")
+              "ll_dict_move_to_first: lost old_index")
     ENTRY = lltype.typeOf(d.entries).TO.OF
     old_entry = d.entries[old_index]
     key = old_entry.key
     value = old_entry.value
     if hasattr(ENTRY, 'f_hash'):
         ll_assert(old_entry.f_hash == hash,
-                  "ll_dict_move_to_beginning: bad hash")
+                  "ll_dict_move_to_first: bad hash")
     _ll_dict_del_entry(d, old_index)
 
     # put the entry at its new position
     ll_assert(not d.entries.valid(idst),
-              "ll_dict_move_to_beginning: overwriting idst")
+              "ll_dict_move_to_first: overwriting idst")
     new_entry = d.entries[idst]
     new_entry.key = key
     new_entry.value = value
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1191,7 +1191,7 @@
     def move_to_end(self, key, last=True):
         "For test_rordereddict"
 
-    def move_to_beginning(self, key):
+    def move_to_first(self, key):
         self.move_to_end(key, last=False)
 
     def copydict(self):
@@ -1260,9 +1260,9 @@
         return builds(Action,
             just('move_to_end'), tuples(sampled_from(self.space.reference)))
 
-    def st_move_to_beginning(self):
+    def st_move_to_first(self):
         return builds(Action,
-            just('move_to_beginning'),
+            just('move_to_first'),
                 tuples(sampled_from(self.space.reference)))
 
     def steps(self):
@@ -1274,7 +1274,7 @@
             return (
                 self.st_setitem() | sampled_from(global_actions) |
                 self.st_updateitem() | self.st_delitem() |
-                self.st_move_to_end() | self.st_move_to_beginning())
+                self.st_move_to_end() | self.st_move_to_first())
         else:
             return (self.st_setitem() | sampled_from(global_actions))
 

From pypy.commits at gmail.com  Sun Feb  5 02:25:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 23:25:51 -0800 (PST)
Subject: [pypy-commit] cffi default: doc fix
Message-ID: <5896d37f.2a99df0a.43838.a5fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2878:3544b1031b51
Date: 2017-02-05 08:25 +0100
http://bitbucket.org/cffi/cffi/changeset/3544b1031b51/

Log:	doc fix

diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -239,7 +239,7 @@
 #endif
 
 PyDoc_STRVAR(ffi_buffer_doc,
-"ffi.buffer(cdata):\n"
+"ffi.buffer(cdata[, byte_size]):\n"
 "Return a read-write buffer object that references the raw C data\n"
 "pointed to by the given 'cdata'.  The 'cdata' must be a pointer or an\n"
 "array.  Can be passed to functions expecting a buffer, or directly\n"

From pypy.commits at gmail.com  Sun Feb  5 02:29:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 04 Feb 2017 23:29:11 -0800 (PST)
Subject: [pypy-commit] pypy default: update to cffi/3544b1031b51
Message-ID: <5896d447.9da0df0a.3bf17.aaed@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89933:8bf21def2904
Date: 2017-02-05 08:25 +0100
http://bitbucket.org/pypy/pypy/changeset/8bf21def2904/

Log:	update to cffi/3544b1031b51

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -93,6 +93,7 @@
             # ctypes backend: attach these constants to the instance
             self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
+        self.buffer = backend.buffer
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
@@ -316,18 +317,18 @@
         """
         return self._backend.unpack(cdata, length)
 
-    def buffer(self, cdata, size=-1):
-        """Return a read-write buffer object that references the raw C data
-        pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
-        an array.  Can be passed to functions expecting a buffer, or directly
-        manipulated with:
-
-            buf[:]          get a copy of it in a regular string, or
-            buf[idx]        as a single character
-            buf[:] = ...
-            buf[idx] = ...  change the content
-        """
-        return self._backend.buffer(cdata, size)
+   #def buffer(self, cdata, size=-1):
+   #    """Return a read-write buffer object that references the raw C data
+   #    pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
+   #    an array.  Can be passed to functions expecting a buffer, or directly
+   #    manipulated with:
+   #
+   #        buf[:]          get a copy of it in a regular string, or
+   #        buf[idx]        as a single character
+   #        buf[:] = ...
+   #        buf[idx] = ...  change the content
+   #    """
+   #    note that 'buffer' is a type, set on this instance by __init__
 
     def from_buffer(self, python_buffer):
         """Return a <cdata 'char[]'> that points to the data of the
@@ -593,11 +594,15 @@
             ensure('extra_link_args', '/MANIFEST')
 
     def set_source(self, module_name, source, source_extension='.c', **kwds):
+        import os
         if hasattr(self, '_assigned_source'):
             raise ValueError("set_source() cannot be called several times "
                              "per ffi object")
         if not isinstance(module_name, basestring):
             raise TypeError("'module_name' must be a string")
+        if os.sep in module_name or (os.altsep and os.altsep in module_name):
+            raise ValueError("'module_name' must not contain '/': use a dotted "
+                             "name to make a 'package.module' location")
         self._assigned_source = (str(module_name), source,
                                  source_extension, kwds)
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -50,7 +50,7 @@
 
         'string': 'func.string',
         'unpack': 'func.unpack',
-        'buffer': 'cbuffer.buffer',
+        'buffer': 'cbuffer.MiniBuffer',
         'memmove': 'func.memmove',
 
         'get_errno': 'cerrno.get_errno',
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -56,19 +56,8 @@
                 e.w_type = space.w_ValueError
             raise
 
-MiniBuffer.typedef = TypeDef(
-    "_cffi_backend.buffer",
-    __len__ = interp2app(MiniBuffer.descr_len),
-    __getitem__ = interp2app(MiniBuffer.descr_getitem),
-    __setitem__ = interp2app(MiniBuffer.descr_setitem),
-    __weakref__ = make_weakref_descr(MiniBuffer),
-    __str__ = interp2app(MiniBuffer.descr_str),
-    )
-MiniBuffer.typedef.acceptable_as_base_class = False
-
-
 @unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
-def buffer(space, w_cdata, size=-1):
+def MiniBuffer___new__(space, w_subtype, w_cdata, size=-1):
     ctype = w_cdata.ctype
     if isinstance(ctype, ctypeptr.W_CTypePointer):
         if size < 0:
@@ -89,3 +78,25 @@
                     "don't know the size pointed to by '%s'", ctype.name)
     ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
     return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
+
+MiniBuffer.typedef = TypeDef(
+    "_cffi_backend.buffer",
+    __new__ = interp2app(MiniBuffer___new__),
+    __len__ = interp2app(MiniBuffer.descr_len),
+    __getitem__ = interp2app(MiniBuffer.descr_getitem),
+    __setitem__ = interp2app(MiniBuffer.descr_setitem),
+    __weakref__ = make_weakref_descr(MiniBuffer),
+    __str__ = interp2app(MiniBuffer.descr_str),
+    __doc__ = """ffi.buffer(cdata[, byte_size]):
+Return a read-write buffer object that references the raw C data
+pointed to by the given 'cdata'.  The 'cdata' must be a pointer or an
+array.  Can be passed to functions expecting a buffer, or directly
+manipulated with:
+
+    buf[:]          get a copy of it in a regular string, or
+    buf[idx]        as a single character
+    buf[:] = ...
+    buf[idx] = ...  change the content
+""",
+    )
+MiniBuffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -265,22 +265,6 @@
         return self.space.wrap(align)
 
 
-    @unwrap_spec(w_cdata=W_CData, size=int)
-    def descr_buffer(self, w_cdata, size=-1):
-        """\
-Return a read-write buffer object that references the raw C data
-ointed to by the given 'cdata'.  The 'cdata' must be a pointer or an
-array.  Can be passed to functions expecting a buffer, or directly
-manipulated with:
-
-    buf[:]          get a copy of it in a regular string, or
-    buf[idx]        as a single character
-    buf[:] = ...
-    buf[idx] = ...  change the content"""
-        #
-        return cbuffer.buffer(self.space, w_cdata, size)
-
-
     @unwrap_spec(w_name=WrappedDefault(None),
                  w_error=WrappedDefault(None),
                  w_onerror=WrappedDefault(None))
@@ -751,6 +735,9 @@
     return space.appexec([], """():
         return type('error', (Exception,), {'__module__': 'ffi'})""")
 
+def make_buffer(space):
+    return space.gettypefor(cbuffer.MiniBuffer)
+
 _extras = get_dict_rtld_constants()
 if sys.platform == 'win32':
     _extras['getwinerror'] = interp2app(W_FFIObject.descr_getwinerror)
@@ -770,7 +757,7 @@
                                      cls=W_FFIObject),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
-        buffer      = interp2app(W_FFIObject.descr_buffer),
+        buffer      = ClassAttr(make_buffer),
         callback    = interp2app(W_FFIObject.descr_callback),
         cast        = interp2app(W_FFIObject.descr_cast),
         def_extern  = interp2app(W_FFIObject.descr_def_extern),
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2288,6 +2288,7 @@
     buf = buffer(c)
     assert repr(buf).startswith('<_cffi_backend.buffer object at 0x')
     assert bytes(buf) == b"hi there\x00"
+    assert type(buf) is buffer
     if sys.version_info < (3,):
         assert str(buf) == "hi there\x00"
         assert unicode(buf) == u+"hi there\x00"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -256,6 +256,8 @@
         ffi = _cffi1_backend.FFI()
         a = ffi.new("signed char[]", [5, 6, 7])
         assert ffi.buffer(a)[:] == '\x05\x06\x07'
+        assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+        assert type(ffi.buffer(a)) is ffi.buffer
 
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1131,6 +1131,7 @@
             b = ffi.buffer(a)
         except NotImplementedError as e:
             py.test.skip(str(e))
+        assert type(b) is ffi.buffer
         content = b[:]
         assert len(content) == len(b) == 2
         if sys.byteorder == 'little':
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -234,6 +234,7 @@
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
     assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+    assert type(ffi.buffer(a)) is ffi.buffer
 
 def test_ffi_from_buffer():
     import array
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -37,6 +37,11 @@
                                       ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
+def test_set_source_no_slashes():
+    ffi = FFI()
+    py.test.raises(ValueError, ffi.set_source, "abc/def", None)
+    py.test.raises(ValueError, ffi.set_source, "abc/def", "C code")
+
 
 def test_type_table_func():
     check_type_table("double sin(double);",

From pypy.commits at gmail.com  Sun Feb  5 05:08:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 02:08:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge dict-move-to-end
Message-ID: <5896f9a9.4c861c0a.32544.1f33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89934:5e325b5c62d9
Date: 2017-02-05 08:29 +0100
http://bitbucket.org/pypy/pypy/changeset/5e325b5c62d9/

Log:	hg merge dict-move-to-end

diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -190,7 +190,7 @@
 from GC issues: if the program "forgets" an unfinished greenlet, it will
 always be collected at the next garbage collection.
 
-.. _documentation of the greenlets: http://packages.python.org/greenlet/
+.. _documentation of the greenlets: https://greenlet.readthedocs.io/
 
 
 Unimplemented features
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -14,7 +14,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
     SomeFloat, SomeIterator, SomePBC, SomeNone, SomeTypeOf, s_ImpossibleValue,
     s_Bool, s_None, s_Int, unionof, add_knowntypedata,
-    SomeWeakRef, SomeUnicodeString, SomeByteArray)
+    SomeWeakRef, SomeUnicodeString, SomeByteArray, SomeOrderedDict)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
 from rpython.annotator.binaryop import _dict_can_only_throw_keyerror
@@ -575,6 +575,13 @@
         pair(self, s_key).delitem()
     method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
 
+class __extend__(SomeOrderedDict):
+
+    def method_move_to_end(self, s_key, s_last):
+        assert s_Bool.contains(s_last)
+        pair(self, s_key).delitem()
+    method_move_to_end.can_only_throw = _dict_can_only_throw_keyerror
+
 @op.contains.register(SomeString)
 @op.contains.register(SomeUnicodeString)
 def contains_String(annotator, string, char):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,24 @@
         return
     d.delitem_with_hash(key, h)
 
+def _untranslated_move_to_end(d, key, last):
+    "NOT_RPYTHON"
+    value = d.pop(key)
+    if last:
+        d[key] = value
+    else:
+        items = d.items()
+        d.clear()
+        d[key] = value
+        d.update(items)
+
+ at specialize.call_location()
+def move_to_end(d, key, last=True):
+    if not we_are_translated():
+        _untranslated_move_to_end(d, key, last)
+        return
+    d.move_to_end(key, last)
+
 # ____________________________________________________________
 
 def import_from_mixin(M, special_methods=['__init__', '__del__']):
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config, try_inline)
+    fetch_translated_config, try_inline, move_to_end)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -679,6 +679,16 @@
     assert f(29) == 0
     interpret(f, [27])
 
+def test_rordereddict_move_to_end():
+    d = OrderedDict()
+    d['key1'] = 'val1'
+    d['key2'] = 'val2'
+    d['key3'] = 'val3'
+    move_to_end(d, 'key1')
+    assert d.items() == [('key2', 'val2'), ('key3', 'val3'), ('key1', 'val1')]
+    move_to_end(d, 'key1', last=False)
+    assert d.items() == [('key1', 'val1'), ('key2', 'val2'), ('key3', 'val3')]
+
 def test_import_from_mixin():
     class M:    # old-style
         def f(self):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,15 @@
         hop.exception_is_here()
         hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
 
+    def rtype_method_move_to_end(self, hop):
+        v_dict, v_key, v_last = hop.inputargs(
+            self, self.key_repr, lltype.Bool)
+        if not self.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)  # record that we know about it
+        hop.exception_is_here()
+        hop.gendirectcall(ll_dict_move_to_end, v_dict, v_key, v_last)
+
+
 class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
 
     def rtype_getitem((r_dict, r_key), hop):
@@ -542,16 +551,18 @@
         ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
         assert False
 
-def ll_call_delete_by_entry_index(d, hash, i):
+def ll_call_delete_by_entry_index(d, hash, i, replace_with):
+    # only called from _ll_dict_del, whose @jit.look_inside_iff
+    # condition should control when we get inside here with the jit
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_BYTE)
     elif fun == FUNC_SHORT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_SHORT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_SHORT)
     elif IS_64BIT and fun == FUNC_INT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_INT)
     elif fun == FUNC_LONG:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_LONG)
     else:
         # can't be still FUNC_MUST_REINDEX here
         ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
@@ -805,13 +816,12 @@
     ll_dict_delitem_with_hash(d, key, d.keyhash(key))
 
 def ll_dict_delitem_with_hash(d, key, hash):
-    index = d.lookup_function(d, key, hash, FLAG_DELETE)
+    index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
-    _ll_dict_del(d, index)
+    _ll_dict_del(d, hash, index)
 
- at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, index):
+def _ll_dict_del_entry(d, index):
     d.entries.mark_deleted(index)
     d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
@@ -823,6 +833,11 @@
     if ENTRIES.must_clear_value:
         entry.value = lltype.nullptr(ENTRY.value.TO)
 
+ at jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, hash, index):
+    ll_call_delete_by_entry_index(d, hash, index, DELETED)
+    _ll_dict_del_entry(d, index)
+
     if d.num_live_items == 0:
         # Dict is now empty.  Reset these fields.
         d.num_ever_used_items = 0
@@ -963,7 +978,6 @@
 
 FLAG_LOOKUP = 0
 FLAG_STORE = 1
-FLAG_DELETE = 2
 
 @specialize.memo()
 def _ll_ptr_to_array_of(T):
@@ -985,8 +999,6 @@
     if index >= VALID_OFFSET:
         checkingkey = entries[index - VALID_OFFSET].key
         if direct_compare and checkingkey == key:
-            if store_flag == FLAG_DELETE:
-                indexes[i] = rffi.cast(T, DELETED)
             return index - VALID_OFFSET   # found the entry
         if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
             # correct hash, maybe the key is e.g. a different pointer to
@@ -1000,8 +1012,6 @@
                     # the compare did major nasty stuff to the dict: start over
                     return ll_dict_lookup(d, key, hash, store_flag, T)
             if found:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET
         deletedslot = -1
     elif index == DELETED:
@@ -1030,8 +1040,6 @@
         elif index >= VALID_OFFSET:
             checkingkey = entries[index - VALID_OFFSET].key
             if direct_compare and checkingkey == key:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET   # found the entry
             if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
                 # correct hash, maybe the key is e.g. a different pointer to
@@ -1044,8 +1052,6 @@
                         # the compare did major nasty stuff to the dict: start over
                         return ll_dict_lookup(d, key, hash, store_flag, T)
                 if found:
-                    if store_flag == FLAG_DELETE:
-                        indexes[i] = rffi.cast(T, DELETED)
                     return index - VALID_OFFSET
         elif deletedslot == -1:
             deletedslot = intmask(i)
@@ -1066,7 +1072,11 @@
         perturb >>= PERTURB_SHIFT
     indexes[i] = rffi.cast(T, index + VALID_OFFSET)
 
-def ll_dict_delete_by_entry_index(d, hash, locate_index, T):
+# the following function is only called from _ll_dict_del, whose
+# @jit.look_inside_iff condition should control when we get inside
+# here with the jit
+ at jit.unroll_safe
+def ll_dict_delete_by_entry_index(d, hash, locate_index, replace_with, T):
     # Another simplified version of ll_dict_lookup() which locates a
     # hashtable entry with the given 'index' stored in it, and deletes it.
     # This *should* be safe against evil user-level __eq__/__hash__
@@ -1083,7 +1093,7 @@
         i = (i << 2) + i + perturb + 1
         i = i & mask
         perturb >>= PERTURB_SHIFT
-    indexes[i] = rffi.cast(T, DELETED)
+    indexes[i] = rffi.cast(T, replace_with)
 
 # ____________________________________________________________
 #
@@ -1253,18 +1263,8 @@
     if hasattr(DICT, 'fnkeyhash'):
         newdict.fnkeyhash = dict.fnkeyhash
 
-    i = 0
-    while i < newdict.num_ever_used_items:
-        d_entry = newdict.entries[i]
-        entry = dict.entries[i]
-        ENTRY = lltype.typeOf(newdict.entries).TO.OF
-        d_entry.key = entry.key
-        if hasattr(ENTRY, 'f_valid'):
-            d_entry.f_valid = entry.f_valid
-        d_entry.value = entry.value
-        if hasattr(ENTRY, 'f_hash'):
-            d_entry.f_hash = entry.f_hash
-        i += 1
+    rgc.ll_arraycopy(dict.entries, newdict.entries, 0, 0,
+                     newdict.num_ever_used_items)
 
     ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
     return newdict
@@ -1390,8 +1390,6 @@
             break
         dic.num_ever_used_items -= 1
 
-    # we must remove the precise entry in the hashtable that points to 'i'
-    ll_call_delete_by_entry_index(dic, entries.hash(i), i)
     return i
 
 def ll_dict_popitem(ELEM, dic):
@@ -1400,21 +1398,139 @@
     r = lltype.malloc(ELEM.TO)
     r.item0 = recast(ELEM.TO.item0, entry.key)
     r.item1 = recast(ELEM.TO.item1, entry.value)
-    _ll_dict_del(dic, i)
+    _ll_dict_del(dic, dic.entries.hash(i), i)
     return r
 
 def ll_dict_pop(dic, key):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value
 
 def ll_dict_pop_default(dic, key, dfl):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         return dfl
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value
+
+def ll_dict_move_to_end(d, key, last):
+    if last:
+        ll_dict_move_to_last(d, key)
+    else:
+        ll_dict_move_to_first(d, key)
+
+def ll_dict_move_to_last(d, key):
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index < 0:
+        raise KeyError
+
+    if old_index == d.num_ever_used_items - 1:
+        return
+
+    # remove the entry at the old position
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    _ll_dict_del_entry(d, old_index)
+
+    # note a corner case: it is possible that 'replace_with' is just too
+    # large to fit in the type T used so far for the index.  But in that
+    # case, the list 'd.entries' is full, and the following call to
+    # _ll_dict_setitem_lookup_done() will necessarily reindex the dict.
+    # So in that case, this value of 'replace_with' should be ignored.
+    ll_call_delete_by_entry_index(d, hash, old_index,
+            replace_with = VALID_OFFSET + d.num_ever_used_items)
+    _ll_dict_setitem_lookup_done(d, key, value, hash, -1)
+
+def ll_dict_move_to_first(d, key):
+    # In this function, we might do a bit more than the strict minimum
+    # of walks over parts of the array, trying to keep the code at least
+    # semi-reasonable, while the goal is still amortized constant-time
+    # over many calls.
+
+    # Call ll_dict_remove_deleted_items() first if there are too many
+    # deleted items.  Not a perfect solution, because lookup_function()
+    # might do random things with the dict and create many new deleted
+    # items.  Still, should be fine, because nothing crucially depends
+    # on this: the goal is to avoid the dictionary's list growing
+    # forever.
+    if d.num_live_items < len(d.entries) // 2 - 16:
+        ll_dict_remove_deleted_items(d)
+
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index <= 0:
+        if old_index < 0:
+            raise KeyError
+        else:
+            return
+
+    # the goal of the following is to set 'idst' to the number of
+    # deleted entries at the beginning, ensuring 'idst > 0'
+    must_reindex = False
+    if d.entries.valid(0):
+        # the first entry is valid, so we need to make room before.
+        new_allocated = _overallocate_entries_len(d.num_ever_used_items)
+        idst = ((new_allocated - d.num_ever_used_items) * 3) // 4
+        ll_assert(idst > 0, "overallocate did not do enough")
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+        rgc.ll_arraycopy(d.entries, newitems, 0, idst, d.num_ever_used_items)
+        d.entries = newitems
+        i = 0
+        while i < idst:
+            d.entries.mark_deleted(i)
+            i += 1
+        d.num_ever_used_items += idst
+        old_index += idst
+        must_reindex = True
+        idst -= 1
+    else:
+        idst = d.lookup_function_no >> FUNC_SHIFT
+        # All entries in range(0, idst) are deleted.  Check if more are
+        while not d.entries.valid(idst):
+            idst += 1
+        if idst == old_index:
+            d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                    (old_index << FUNC_SHIFT))
+            return
+        idst -= 1
+        d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                (idst << FUNC_SHIFT))
+
+    # remove the entry at the old position
+    ll_assert(d.entries.valid(old_index),
+              "ll_dict_move_to_first: lost old_index")
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    if hasattr(ENTRY, 'f_hash'):
+        ll_assert(old_entry.f_hash == hash,
+                  "ll_dict_move_to_first: bad hash")
+    _ll_dict_del_entry(d, old_index)
+
+    # put the entry at its new position
+    ll_assert(not d.entries.valid(idst),
+              "ll_dict_move_to_first: overwriting idst")
+    new_entry = d.entries[idst]
+    new_entry.key = key
+    new_entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        new_entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        new_entry.f_valid = True
+    d.num_live_items += 1
+
+    # fix the index
+    if must_reindex:
+        ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+    else:
+        ll_call_delete_by_entry_index(d, hash, old_index,
+                replace_with = VALID_OFFSET + idst)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1188,6 +1188,12 @@
         assert not self.ll_contains(self.l_dict, ll_key)
         self.removed_keys.append(key)
 
+    def move_to_end(self, key, last=True):
+        "For test_rordereddict"
+
+    def move_to_first(self, key):
+        self.move_to_end(key, last=False)
+
     def copydict(self):
         self.l_dict = self.ll_copy(self.l_dict)
         assert self.ll_len(self.l_dict) == len(self.reference)
@@ -1250,6 +1256,15 @@
         return builds(Action,
             just('delitem'), tuples(sampled_from(self.space.reference)))
 
+    def st_move_to_end(self):
+        return builds(Action,
+            just('move_to_end'), tuples(sampled_from(self.space.reference)))
+
+    def st_move_to_first(self):
+        return builds(Action,
+            just('move_to_first'),
+                tuples(sampled_from(self.space.reference)))
+
     def steps(self):
         if not self.space:
             return builds(Action, just('setup'), tuples(st_keys, st_values))
@@ -1258,7 +1273,8 @@
         if self.space.reference:
             return (
                 self.st_setitem() | sampled_from(global_actions) |
-                self.st_updateitem() | self.st_delitem())
+                self.st_updateitem() | self.st_delitem() |
+                self.st_move_to_end() | self.st_move_to_first())
         else:
             return (self.st_setitem() | sampled_from(global_actions))
 
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -1,7 +1,8 @@
 import py
+import random
 from collections import OrderedDict
 
-from hypothesis import settings
+from hypothesis import settings, given, strategies
 from hypothesis.stateful import run_state_machine_as_test
 
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -145,14 +146,18 @@
         ll_d = rordereddict.ll_newdict(DICT)
         rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
         rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
-        ITER = rordereddict.get_ll_dictiter(lltype.Ptr(DICT))
+        assert [hlstr(entry.key) for entry in self._ll_iter(ll_d)] == ["k", "j"]
+
+    def _ll_iter(self, ll_d):
+        ITER = rordereddict.get_ll_dictiter(lltype.typeOf(ll_d))
         ll_iter = rordereddict.ll_dictiter(ITER, ll_d)
         ll_dictnext = rordereddict._ll_dictnext
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "k"
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "j"
-        py.test.raises(StopIteration, ll_dictnext, ll_iter)
+        while True:
+            try:
+                num = ll_dictnext(ll_iter)
+            except StopIteration:
+                break
+            yield ll_d.entries[num]
 
     def test_popitem(self):
         DICT = self._get_str_dict()
@@ -337,6 +342,31 @@
             num_nonfrees += (got > 0)
         assert d.resize_counter <= idx.getlength() * 2 - num_nonfrees * 3
 
+    @given(strategies.lists(strategies.integers(min_value=1, max_value=5)))
+    def test_direct_move_to_end(self, lst):
+        DICT = self._get_int_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, 1, 11)
+        rordereddict.ll_dict_setitem(ll_d, 2, 22)
+        def content():
+            return [(entry.key, entry.value) for entry in self._ll_iter(ll_d)]
+        for case in lst:
+            if case == 1:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, True)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 2:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, True)
+                assert content() == [(1, 11), (2, 22)]
+            elif case == 3:
+                py.test.raises(KeyError, rordereddict.ll_dict_move_to_end,
+                                                 ll_d, 3, True)
+            elif case == 4:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, False)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 5:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, False)
+                assert content() == [(1, 11), (2, 22)]
+
 
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
@@ -369,10 +399,29 @@
         res = self.interpret(func, [5])
         assert res == 6
 
+    def test_move_to_end(self):
+        def func():
+            d1 = OrderedDict()
+            d1['key1'] = 'value1'
+            d1['key2'] = 'value2'
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key1')
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key2')
+                assert d1.keys() == ['key1', 'key2']
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key2', last=False)
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key1', last=False)
+                assert d1.keys() == ['key1', 'key2']
+        func()
+        self.interpret(func, [])
+
 
 class ODictSpace(MappingSpace):
     MappingRepr = rodct.OrderedDictRepr
     new_reference = OrderedDict
+    moved_around = False
     ll_getitem = staticmethod(rodct.ll_dict_getitem)
     ll_setitem = staticmethod(rodct.ll_dict_setitem)
     ll_delitem = staticmethod(rodct.ll_dict_delitem)
@@ -417,9 +466,23 @@
     def removeindex(self):
         # remove the index, as done during translation for prebuilt dicts
         # (but cannot be done if we already removed a key)
-        if not self.removed_keys:
+        if not self.removed_keys and not self.moved_around:
             rodct.ll_no_initial_index(self.l_dict)
 
+    def move_to_end(self, key, last=True):
+        ll_key = self.ll_key(key)
+        rodct.ll_dict_move_to_end(self.l_dict, ll_key, last)
+        value = self.reference.pop(key)
+        if last:
+            self.reference[key] = value
+        else:
+            items = self.reference.items()
+            self.reference.clear()
+            self.reference[key] = value
+            self.reference.update(items)
+        # prevent ll_no_initial_index()
+        self.moved_around = True
+
     def fullcheck(self):
         # overridden to also check key order
         assert self.ll_len(self.l_dict) == len(self.reference)

From pypy.commits at gmail.com  Sun Feb  5 05:08:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 02:08:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add __pypy__.move_to_end(),
 similar to __pypy__.reversed_dict().
Message-ID: <5896f9ab.52a5df0a.6e805.cd29@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89935:6974fd5f5c47
Date: 2017-02-05 10:57 +0100
http://bitbucket.org/pypy/pypy/changeset/6974fd5f5c47/

Log:	Add __pypy__.move_to_end(), similar to __pypy__.reversed_dict().

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -78,6 +78,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
+        'move_to_end'               : 'interp_dict.move_to_end',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -44,3 +44,18 @@
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_reversed_dict(space)
+
+ at unwrap_spec(last=bool)
+def move_to_end(space, w_obj, w_key, last=True):
+    """Move the key in a dictionary object into the first or last position.
+
+    This is a __pypy__ function instead of being simply done by calling
+    dict.move_to_end(), for CPython compatibility: dictionaries are only
+    ordered on PyPy.  You should use the collections.OrderedDict class for
+    cases where ordering is important.  That class implements the
+    move_to_end() method by calling __pypy__.move_to_end().
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_move_to_end(space, w_key, last)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -232,6 +232,37 @@
             w_keys = self.w_keys()
             return space.call_method(w_keys, '__reversed__')
 
+    def nondescr_move_to_end(self, space, w_key, last_flag):
+        """Not exposed directly to app-level, but via __pypy__.move_to_end().
+        """
+        strategy = self.get_strategy()
+        if strategy.has_move_to_end:
+            strategy.move_to_end(self, w_key, last_flag)
+        else:
+            # fall-back
+            w_value = self.getitem(w_key)
+            if w_value is None:
+                space.raise_key_error(w_key)
+            else:
+                self.delitem(w_key)
+                if last_flag:
+                    self.setitem(w_key, w_value)
+                else:
+                    # *very slow* fall-back
+                    keys_w = []
+                    values_w = []
+                    iteratorimplementation = self.iteritems()
+                    while True:
+                        w_k, w_v = iteratorimplementation.next_item()
+                        if w_k is None:
+                            break
+                        keys_w.append(w_k)
+                        values_w.append(w_v)
+                    self.clear()
+                    self.setitem(w_key, w_value)
+                    for i in range(len(keys_w)):
+                        self.setitem(keys_w[i], values_w[i])
+
     def descr_clear(self, space):
         """D.clear() -> None.  Remove all items from D."""
         self.clear()
@@ -499,7 +530,9 @@
         raise NotImplementedError
 
     has_iterreversed = False
-    # no 'getiterreversed': no default implementation available
+    has_move_to_end = False
+    # no 'getiterreversed' and no 'move_to_end': no default
+    # implementation available
 
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         iteritems = self.iteritems(w_dict)
@@ -783,6 +816,9 @@
         dictimpl.iterreversed = iterreversed
         dictimpl.has_iterreversed = True
 
+    if hasattr(dictimpl, 'move_to_end'):
+        dictimpl.has_move_to_end = True
+
     @jit.look_inside_iff(lambda self, w_dict, w_updatedict:
                          w_dict_unrolling_heuristic(w_dict))
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
@@ -962,6 +998,15 @@
     def getiterreversed(self, w_dict):
         return objectmodel.reversed_dict(self.unerase(w_dict.dstorage))
 
+    def move_to_end(self, w_dict, w_key, last_flag):
+        if self.is_correct_type(w_key):
+            d = self.unerase(w_dict.dstorage)
+            key = self.unwrap(w_key)
+            objectmodel.move_to_end(d, key, last_flag)
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.nondescr_move_to_end(w_dict.space, w_key, last_flag)
+
     def prepare_update(self, w_dict, num_extra):
         objectmodel.prepare_dict_update(self.unerase(w_dict.dstorage),
                                         num_extra)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -270,6 +270,29 @@
         del d[key]
         raises(RuntimeError, next, it)
 
+    def test_move_to_end(self):
+        import __pypy__
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo')
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=True)
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=False)
+        def kwdict(**k):
+            return k
+        for last in [False, True]:
+            for d, key in [({1: 2, 3: 4, 5: 6}, 3),
+                           ({"a": 5, "b": 2, "c": 6}, "b"),
+                           (kwdict(d=7, e=8, f=9), "e")]:
+                other_keys = [k for k in d if k != key]
+                __pypy__.move_to_end(d, key, last=last)
+                if not self.on_pypy:
+                    # when running tests on CPython, the underlying
+                    # dicts are not ordered.  We don't get here if
+                    # we're running tests on PyPy or with -A.
+                    assert set(d.keys()) == set(other_keys + [key])
+                elif last:
+                    assert list(d) == other_keys + [key]
+                else:
+                    assert list(d) == [key] + other_keys
+
     def test_keys(self):
         d = {1: 2, 3: 4}
         kys = list(d.keys())

From pypy.commits at gmail.com  Sun Feb  5 05:08:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 02:08:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: PyPy's simplified version of OrderedDict
Message-ID: <5896f9ac.cda0df0a.dace6.ae85@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89936:5764bb3f54a2
Date: 2017-02-05 11:08 +0100
http://bitbucket.org/pypy/pypy/changeset/5764bb3f54a2/

Log:	PyPy's simplified version of OrderedDict

diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -439,3 +439,8 @@
         return (type(self), (self.default_factory,), None, None,
                 iter(self.items()))
 
+
+try:
+    from _pypy_collections import OrderedDict
+except ImportError:
+    pass
diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_collections.py
@@ -0,0 +1,69 @@
+from __pypy__ import reversed_dict, move_to_end
+from reprlib import recursive_repr as _recursive_repr
+
+class OrderedDict(dict):
+    '''Dictionary that remembers insertion order.
+
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
+
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
+
+    def __reversed__(self):
+        return reversed_dict(self)
+
+    def popitem(self, last=True):
+        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+        Pairs are returned in LIFO order if last is true or FIFO order if false.
+
+        '''
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
+
+    def move_to_end(self, key, last=True):
+        '''Move an existing element to the end (or beginning if last==False).
+
+        Raises KeyError if the element does not exist.
+        When last=True, acts like a fast version of self[key]=self.pop(key).
+
+        '''
+        return move_to_end(self, key, last)
+
+    @_recursive_repr()
+    def __repr__(self):
+        'od.__repr__() <==> repr(od)'
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self.items()))
+
+    def __reduce__(self):
+        'Return state information for pickling'
+        inst_dict = vars(self).copy()
+        return self.__class__, (), inst_dict or None, None, iter(self.items())
+
+    def copy(self):
+        'od.copy() -> a shallow copy of od'
+        return self.__class__(self)
+
+    def __eq__(self, other):
+        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
+        while comparison to a regular mapping is order-insensitive.
+
+        '''
+        if isinstance(other, OrderedDict):
+            return dict.__eq__(self, other) and all(map(_eq, self, other))
+        return dict.__eq__(self, other)
+
+    __ne__ = object.__ne__
diff --git a/pypy/module/_collections/__init__.py b/pypy/module/_collections/__init__.py
--- a/pypy/module/_collections/__init__.py
+++ b/pypy/module/_collections/__init__.py
@@ -25,3 +25,15 @@
         space = self.space
         space.getattr(self, space.wrap('defaultdict'))  # force importing
         space.delattr(self, space.wrap('__missing__'))
+
+    def startup(self, space):
+        # OrderedDict is normally present, but in some cases the line
+        # "from __pypy__ import reversed_dict, move_to_end" from
+        # _pypy_collections.py raises
+        space.appexec([space.wrap(self)], """(mod):
+            try:
+                from _pypy_collections import OrderedDict
+                mod.OrderedDict = OrderedDict
+            except ImportError:
+                pass
+        """)
diff --git a/pypy/module/_collections/test/test_ordereddict.py b/pypy/module/_collections/test/test_ordereddict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_collections/test/test_ordereddict.py
@@ -0,0 +1,8 @@
+
+class AppTestBasic:
+    spaceconfig = dict(usemodules=['_collections'])
+
+    def test_ordereddict_present(self):
+        from _collections import OrderedDict
+        assert issubclass(OrderedDict, dict)
+        assert hasattr(OrderedDict, 'move_to_end')

From pypy.commits at gmail.com  Sun Feb  5 05:08:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 02:08:49 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <5896f9b1.c53f1c0a.e9477.1f32@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r851:f22c38f24a8b
Date: 2017-02-05 11:08 +0100
http://bitbucket.org/pypy/pypy.org/changeset/f22c38f24a8b/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66561 of $105000 (63.4%)
+   $66570 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2391 left
+    we have $2399 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sun Feb  5 05:41:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 02:41:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: missing import
Message-ID: <58970156.1282df0a.bb340.cebe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89937:fb91b76b7210
Date: 2017-02-05 11:40 +0100
http://bitbucket.org/pypy/pypy/changeset/fb91b76b7210/

Log:	missing import

diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
--- a/lib_pypy/_pypy_collections.py
+++ b/lib_pypy/_pypy_collections.py
@@ -1,4 +1,5 @@
 from __pypy__ import reversed_dict, move_to_end
+from _operator import eq as _eq
 from reprlib import recursive_repr as _recursive_repr
 
 class OrderedDict(dict):

From pypy.commits at gmail.com  Sun Feb  5 09:34:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 06:34:05 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: done
Message-ID: <589737dd.6787df0a.b4168.10d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5773:811543e4ff04
Date: 2017-02-05 15:33 +0100
http://bitbucket.org/pypy/extradoc/changeset/811543e4ff04/

Log:	done

diff --git a/planning/py3.5/milestone-2-progress.rst b/planning/py3.5/milestone-2-progress.rst
--- a/planning/py3.5/milestone-2-progress.rst
+++ b/planning/py3.5/milestone-2-progress.rst
@@ -17,6 +17,7 @@
 * collections.py: ``OrderedDict`` should again be a thin wrapper over
   ``dict``.  The main pain point is ``move_to_end(last=False)``.  See
   https://mail.python.org/pipermail/python-dev/2016-August/145837.html
+  [DONE]
 
 * compare ``dir(posix)`` on py3.5 and cpython 3.5.
 

From pypy.commits at gmail.com  Sun Feb  5 11:14:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 08:14:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Normalize the PyMem_Xxx functions and
 macros to CPython 3.5's situation
Message-ID: <58974f53.c53f1c0a.e9477.908f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89938:319cd6335518
Date: 2017-02-05 17:13 +0100
http://bitbucket.org/pypy/pypy/changeset/319cd6335518/

Log:	Normalize the PyMem_Xxx functions and macros to CPython 3.5's
	situation

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -612,6 +612,9 @@
     'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag',
     'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory',
     '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext',
+
+    'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree',
+    'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free',
 ]
 TYPES = {}
 FORWARD_DECLS = []
diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -7,17 +7,22 @@
 extern "C" {
 #endif
 
-#define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
-#define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
-#define PyMem_FREE		free
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+#endif
 
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-#define PyMem_Free  PyMem_FREE
-#define PyMem_Realloc  PyMem_REALLOC
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
-#define PyMem_RawMalloc PyMem_Malloc
-#define PyMem_RawFree PyMem_Free
-#define PyMem_RawRealloc PyMem_Realloc
+#define PyMem_MALLOC(n)         PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n)     PyMem_Realloc(p, n)
+#define PyMem_FREE(p)           PyMem_Free(p)
+
 
 /*
  * Type-oriented memory interface
diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c
--- a/pypy/module/cpyext/src/pymem.c
+++ b/pypy/module/cpyext/src/pymem.c
@@ -1,6 +1,86 @@
 #include <Python.h>
 
-void * PyMem_Malloc(size_t n)
+void *
+PyMem_RawMalloc(size_t size)
 {
-    return malloc((n) ? (n) : 1);
+    /*
+     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
+     * Most python internals blindly use a signed Py_ssize_t to track
+     * things without checking for overflows or negatives.
+     * As size_t is unsigned, checking for size < 0 is not required.
+     */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return malloc(size);
 }
+
+void *
+PyMem_RawCalloc(size_t nelem, size_t elsize)
+{
+    /* see PyMem_RawMalloc() */
+    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+        return NULL;
+    /* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL
+       for calloc(0, 0), which would be treated as an error. Some platforms
+       would return a pointer with no memory behind it, which would break
+       pymalloc.  To solve these problems, allocate an extra byte. */
+    if (nelem == 0 || elsize == 0) {
+        nelem = 1;
+        elsize = 1;
+    }
+    return calloc(nelem, elsize);
+}
+
+void*
+PyMem_RawRealloc(void *ptr, size_t size)
+{
+    /* see PyMem_RawMalloc() */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return realloc(ptr, size);
+}
+
+void PyMem_RawFree(void *ptr)
+{
+    free(ptr);
+}
+
+
+/* the PyMem_Xxx functions are the same as PyMem_RawXxx in PyPy, for now */
+void *PyMem_Malloc(size_t size)
+{
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return malloc(size);
+}
+
+void *PyMem_Calloc(size_t nelem, size_t elsize)
+{
+    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+        return NULL;
+    if (nelem == 0 || elsize == 0) {
+        nelem = 1;
+        elsize = 1;
+    }
+    return calloc(nelem, elsize);
+}
+
+void* PyMem_Realloc(void *ptr, size_t size)
+{
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return realloc(ptr, size);
+}
+
+void PyMem_Free(void *ptr)
+{
+    free(ptr);
+}
diff --git a/pypy/module/cpyext/test/test_pymem.py b/pypy/module/cpyext/test/test_pymem.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_pymem.py
@@ -0,0 +1,34 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class AppTestPyMem(AppTestCpythonExtensionBase):
+    def test_pymem_alloc(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+                int *a, *b;
+                a = PyMem_RawCalloc(4, 50);
+                if (a[49] != 0) {
+                    PyErr_SetString(PyExc_ValueError, "1");
+                    return NULL;
+                }
+                a[49] = 123456;
+                b = PyMem_RawRealloc(a, 2000);
+                b[499] = 789123;
+                PyMem_RawFree(b);
+
+                a = PyMem_Calloc(4, 50);
+                if (a[49] != 0) {
+                    PyErr_SetString(PyExc_ValueError, "2");
+                    return NULL;
+                }
+                a[49] = 123456;
+                b = PyMem_Realloc(a, 2000);
+                b[499] = 789123;
+                PyMem_Free(b);
+
+                Py_RETURN_NONE;
+             """),
+            ])
+        res = module.test()
+        assert res is None

From pypy.commits at gmail.com  Sun Feb  5 11:17:08 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 05 Feb 2017 08:17:08 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: Add getformat() and
 getitemsize() to RawFFIBuffer
Message-ID: <58975004.0c8e1c0a.8d09a.9861@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r89939:82e6578296c4
Date: 2017-02-05 16:15 +0000
http://bitbucket.org/pypy/pypy/changeset/82e6578296c4/

Log:	Add getformat() and getitemsize() to RawFFIBuffer

diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -91,6 +91,8 @@
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
+        self.fmt = shape.itemcode
+        self.itemsize = shape.size
 
     def descr_repr(self, space):
         addr = rffi.cast(lltype.Unsigned, self.ll_buffer)
@@ -105,8 +107,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, write_ptr, self.ll_buffer, num,
-                     self.shape.itemcode, w_value)
+        unwrap_value(space, write_ptr, self.ll_buffer, num, self.fmt, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
         try:
@@ -123,8 +124,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, read_ptr, self.ll_buffer, num,
-                          self.shape.itemcode)
+        return wrap_value(space, read_ptr, self.ll_buffer, num, self.fmt)
 
     def descr_getitem(self, space, w_index):
         try:
@@ -141,19 +141,16 @@
 
     @unwrap_spec(num=int)
     def descr_itemaddress(self, space, num):
-        itemsize = self.shape.size
-        ptr = rffi.ptradd(self.ll_buffer, itemsize * num)
+        ptr = rffi.ptradd(self.ll_buffer, self.itemsize * num)
         return space.wrap(rffi.cast(lltype.Unsigned, ptr))
 
     def getrawsize(self):
-        itemsize = self.shape.size
-        return itemsize * self.length
+        return self.itemsize * self.length
 
     def decodeslice(self, space, w_slice):
         if not space.isinstance_w(w_slice, space.w_slice):
             raise oefmt(space.w_TypeError, "index must be int or slice")
-        letter = self.shape.itemcode
-        if letter != 'c':
+        if self.fmt != 'c':
             raise oefmt(space.w_TypeError, "only 'c' arrays support slicing")
         w_start = space.getattr(w_slice, space.wrap('start'))
         w_stop = space.getattr(w_slice, space.wrap('stop'))
@@ -193,7 +190,8 @@
             ll_buffer[start + i] = value[i]
 
     def buffer_w_ex(self, space, flags):
-        return self.buffer_w(space, flags), self.shape.itemcode, self.shape.size
+        buf = self.buffer_w(space, flags)
+        return buf, buf.getformat(), buf.getitemsize()
 
 
 W_ArrayInstance.typedef = TypeDef(
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -14,6 +14,12 @@
     def getlength(self):
         return self.datainstance.getrawsize()
 
+    def getformat(self):
+        return self.datainstance.fmt
+
+    def getitemsize(self):
+        return self.datainstance.itemsize
+
     def getitem(self, index):
         ll_buffer = self.datainstance.ll_buffer
         return ll_buffer[index]
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -337,6 +337,8 @@
 
 
 class W_DataInstance(W_Root):
+    fmt = 'B'
+    itemsize = 1
     def __init__(self, space, size, address=r_uint(0)):
         if address:
             self.ll_buffer = rffi.cast(rffi.VOIDP, address)

From pypy.commits at gmail.com  Sun Feb  5 11:17:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 08:17:56 -0800 (PST)
Subject: [pypy-commit] pypy default: silence a warning
Message-ID: <58975034.d3811c0a.be7ac.92d0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89940:c763e1ab0f07
Date: 2017-02-05 17:17 +0100
http://bitbucket.org/pypy/pypy/changeset/c763e1ab0f07/

Log:	silence a warning

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -602,7 +602,7 @@
 
 if rposix.HAVE_FSTATAT:
     from rpython.rlib.rposix import AT_FDCWD, AT_SYMLINK_NOFOLLOW
-    c_fstatat = rffi.llexternal('fstatat',
+    c_fstatat = rffi.llexternal('fstatat64' if _LINUX else 'fstatat',
         [rffi.INT, rffi.CCHARP, STAT_STRUCT, rffi.INT], rffi.INT,
         compilation_info=compilation_info,
         save_err=rffi.RFFI_SAVE_ERRNO, macro=True)

From pypy.commits at gmail.com  Sun Feb  5 11:31:57 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 05 Feb 2017 08:31:57 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: Add W_ArrayBase.buffer_w()
Message-ID: <5897537d.b296df0a.eab4e.36d5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r89942:1ed5b196b55d
Date: 2017-02-05 16:31 +0000
http://bitbucket.org/pypy/pypy/changeset/1ed5b196b55d/

Log:	Add W_ArrayBase.buffer_w()

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -256,8 +256,12 @@
         if oldbuffer:
             lltype.free(oldbuffer, flavor='raw')
 
+    def buffer_w(self, space, flags):
+        return ArrayBuffer(self, False)
+
     def buffer_w_ex(self, space, flags):
-        return ArrayBuffer(self, False), self.typecode, self.itemsize
+        buf = self.buffer_w(space, flags)
+        return buf, buf.getformat(), buf.getitemsize()
 
     def descr_append(self, space, w_x):
         """ append(x)

From pypy.commits at gmail.com  Sun Feb  5 11:22:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 08:22:25 -0800 (PST)
Subject: [pypy-commit] pypy default: silence a warning
Message-ID: <58975141.d185df0a.ad9ab.29ec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89941:587b40019180
Date: 2017-02-05 17:21 +0100
http://bitbucket.org/pypy/pypy/changeset/587b40019180/

Log:	silence a warning

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1069,7 +1069,8 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI;
+    struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     global_objects = []

From pypy.commits at gmail.com  Sun Feb  5 11:44:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 08:44:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: PyErr_WarnFormat
Message-ID: <5897568b.04ba1c0a.56387.9ad1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89943:386cfc419af7
Date: 2017-02-05 17:44 +0100
http://bitbucket.org/pypy/pypy/changeset/386cfc419af7/

Log:	PyErr_WarnFormat

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -571,6 +571,7 @@
     '_Py_BuildValue_SizeT', '_Py_VaBuildValue_SizeT',
 
     'PyErr_Format', 'PyErr_NewException', 'PyErr_NewExceptionWithDoc',
+    'PyErr_WarnFormat',
     'PySys_WriteStdout', 'PySys_WriteStderr',
 
     'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction',
@@ -1321,6 +1322,7 @@
                          source_dir / "bytesobject.c",
                          source_dir / "complexobject.c",
                          source_dir / "import.c",
+                         source_dir / "_warnings.c",
                          ]
 
 def build_eci(code, use_micronumpy=False, translating=False):
diff --git a/pypy/module/cpyext/include/warnings.h b/pypy/module/cpyext/include/warnings.h
--- a/pypy/module/cpyext/include/warnings.h
+++ b/pypy/module/cpyext/include/warnings.h
@@ -6,6 +6,9 @@
 
 #define PyErr_WarnPy3k(msg, stacklevel) 0
 
+PyAPI_FUNC(int) PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
+                                 const char *format, ...);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/src/_warnings.c b/pypy/module/cpyext/src/_warnings.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/_warnings.c
@@ -0,0 +1,25 @@
+#include <Python.h>
+
+int
+PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
+                 const char *format, ...)
+{
+    int ret;
+    PyObject *message;
+    va_list vargs;
+
+#ifdef HAVE_STDARG_PROTOTYPES
+    va_start(vargs, format);
+#else
+    va_start(vargs);
+#endif
+    message = PyUnicode_FromFormatV(format, vargs);
+    if (message != NULL) {
+        ret = PyErr_WarnEx(category, PyUnicode_AsUTF8(message), stack_level);
+        Py_DECREF(message);
+    }
+    else
+        ret = -1;
+    va_end(vargs);
+    return ret;
+}
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -405,14 +405,6 @@
     (sys.getfilesystemencoding())."""
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARP, ], rffi.INT_real, error=-1)
-def PyErr_WarnFormat(space, category, stack_level, format, ):
-    """Function similar to PyErr_WarnEx(), but use
-    PyUnicode_FromFormat() to format the warning message.  format is
-    an ASCII-encoded string.
-    """
-    raise NotImplementedError
-
 
 @cpython_api([rffi.INT_real], rffi.INT_real, error=-1)
 def PySignal_SetWakeupFd(space, fd):
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -428,3 +428,18 @@
             assert orig_exc_info == reset_sys_exc_info
             assert new_exc_info == (new_exc.__class__, new_exc, None)
             assert new_exc_info == new_sys_exc_info
+
+    def test_PyErr_WarnFormat(self):
+        import warnings
+
+        module = self.import_extension('foo', [
+                ("test", "METH_NOARGS",
+                 '''
+                 PyErr_WarnFormat(PyExc_UserWarning, 1, "foo %d bar", 42);
+                 Py_RETURN_NONE;
+                 '''),
+                ])
+        with warnings.catch_warnings(record=True) as l:
+            module.test()
+        assert len(l) == 1
+        assert "foo 42 bar" in str(l[0])

From pypy.commits at gmail.com  Sun Feb  5 11:53:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 08:53:49 -0800 (PST)
Subject: [pypy-commit] pypy default: PyGen_Check(), PyGen_CheckExact()
Message-ID: <5897589d.0f96df0a.7e973.3dc2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89944:7a5c63c2feec
Date: 2017-02-05 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/7a5c63c2feec/

Log:	PyGen_Check(), PyGen_CheckExact()

diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/genobject.py
@@ -0,0 +1,5 @@
+from pypy.interpreter.generator import GeneratorIterator
+from pypy.module.cpyext.api import build_type_checkers
+
+
+PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -695,17 +695,6 @@
     extension modules."""
     raise NotImplementedError
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_Check(space, ob):
-    """Return true if ob is a generator object; ob must not be NULL."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_CheckExact(space, ob):
-    """Return true if ob's type is PyGen_Type is a generator object; ob must not
-    be NULL."""
-    raise NotImplementedError
-
 @cpython_api([PyFrameObject], PyObject)
 def PyGen_New(space, frame):
     """Create and return a new generator object based on the frame object. A
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -0,0 +1,15 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.genobject import PyGen_Check, PyGen_CheckExact
+
+
+class TestGenObject(BaseApiTest):
+    def test_genobject(self, space):
+        w_geniter = space.appexec([], """():
+            def f():
+                yield 42
+            return f()
+        """)
+        assert PyGen_Check(space, w_geniter)
+        assert PyGen_CheckExact(space, w_geniter)
+        assert not PyGen_Check(space, space.wrap(2))
+        assert not PyGen_CheckExact(space, space.wrap("b"))

From pypy.commits at gmail.com  Sun Feb  5 11:56:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 08:56:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58975926.10941c0a.65f0.a029@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89945:6a45f8963c90
Date: 2017-02-05 17:55 +0100
http://bitbucket.org/pypy/pypy/changeset/6a45f8963c90/

Log:	hg merge default

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -93,6 +93,7 @@
             # ctypes backend: attach these constants to the instance
             self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
+        self.buffer = backend.buffer
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
@@ -316,18 +317,18 @@
         """
         return self._backend.unpack(cdata, length)
 
-    def buffer(self, cdata, size=-1):
-        """Return a read-write buffer object that references the raw C data
-        pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
-        an array.  Can be passed to functions expecting a buffer, or directly
-        manipulated with:
-
-            buf[:]          get a copy of it in a regular string, or
-            buf[idx]        as a single character
-            buf[:] = ...
-            buf[idx] = ...  change the content
-        """
-        return self._backend.buffer(cdata, size)
+   #def buffer(self, cdata, size=-1):
+   #    """Return a read-write buffer object that references the raw C data
+   #    pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
+   #    an array.  Can be passed to functions expecting a buffer, or directly
+   #    manipulated with:
+   #
+   #        buf[:]          get a copy of it in a regular string, or
+   #        buf[idx]        as a single character
+   #        buf[:] = ...
+   #        buf[idx] = ...  change the content
+   #    """
+   #    note that 'buffer' is a type, set on this instance by __init__
 
     def from_buffer(self, python_buffer):
         """Return a <cdata 'char[]'> that points to the data of the
@@ -593,11 +594,15 @@
             ensure('extra_link_args', '/MANIFEST')
 
     def set_source(self, module_name, source, source_extension='.c', **kwds):
+        import os
         if hasattr(self, '_assigned_source'):
             raise ValueError("set_source() cannot be called several times "
                              "per ffi object")
         if not isinstance(module_name, basestring):
             raise TypeError("'module_name' must be a string")
+        if os.sep in module_name or (os.altsep and os.altsep in module_name):
+            raise ValueError("'module_name' must not contain '/': use a dotted "
+                             "name to make a 'package.module' location")
         self._assigned_source = (str(module_name), source,
                                  source_extension, kwds)
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -50,7 +50,7 @@
 
         'string': 'func.string',
         'unpack': 'func.unpack',
-        'buffer': 'cbuffer.buffer',
+        'buffer': 'cbuffer.MiniBuffer',
         'memmove': 'func.memmove',
 
         'get_errno': 'cerrno.get_errno',
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -75,18 +75,8 @@
             self.buffer.setslice(start, value.as_str())
 
 
-MiniBuffer.typedef = TypeDef(
-    "_cffi_backend.buffer",
-    __len__ = interp2app(MiniBuffer.descr_len),
-    __getitem__ = interp2app(MiniBuffer.descr_getitem),
-    __setitem__ = interp2app(MiniBuffer.descr_setitem),
-    __weakref__ = make_weakref_descr(MiniBuffer),
-    )
-MiniBuffer.typedef.acceptable_as_base_class = False
-
-
 @unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
-def buffer(space, w_cdata, size=-1):
+def MiniBuffer___new__(space, w_subtype, w_cdata, size=-1):
     ctype = w_cdata.ctype
     if isinstance(ctype, ctypeptr.W_CTypePointer):
         if size < 0:
@@ -107,3 +97,24 @@
                     "don't know the size pointed to by '%s'", ctype.name)
     ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
     return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
+
+MiniBuffer.typedef = TypeDef(
+    "_cffi_backend.buffer",
+    __new__ = interp2app(MiniBuffer___new__),
+    __len__ = interp2app(MiniBuffer.descr_len),
+    __getitem__ = interp2app(MiniBuffer.descr_getitem),
+    __setitem__ = interp2app(MiniBuffer.descr_setitem),
+    __weakref__ = make_weakref_descr(MiniBuffer),
+    __doc__ = """ffi.buffer(cdata[, byte_size]):
+Return a read-write buffer object that references the raw C data
+pointed to by the given 'cdata'.  The 'cdata' must be a pointer or an
+array.  Can be passed to functions expecting a buffer, or directly
+manipulated with:
+
+    buf[:]          get a copy of it in a regular string, or
+    buf[idx]        as a single character
+    buf[:] = ...
+    buf[idx] = ...  change the content
+""",
+    )
+MiniBuffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -265,22 +265,6 @@
         return self.space.wrap(align)
 
 
-    @unwrap_spec(w_cdata=W_CData, size=int)
-    def descr_buffer(self, w_cdata, size=-1):
-        """\
-Return a read-write buffer object that references the raw C data
-ointed to by the given 'cdata'.  The 'cdata' must be a pointer or an
-array.  Can be passed to functions expecting a buffer, or directly
-manipulated with:
-
-    buf[:]          get a copy of it in a regular string, or
-    buf[idx]        as a single character
-    buf[:] = ...
-    buf[idx] = ...  change the content"""
-        #
-        return cbuffer.buffer(self.space, w_cdata, size)
-
-
     @unwrap_spec(w_name=WrappedDefault(None),
                  w_error=WrappedDefault(None),
                  w_onerror=WrappedDefault(None))
@@ -751,6 +735,9 @@
     return space.appexec([], """():
         return type('error', (Exception,), {'__module__': 'ffi'})""")
 
+def make_buffer(space):
+    return space.gettypefor(cbuffer.MiniBuffer)
+
 _extras = get_dict_rtld_constants()
 if sys.platform == 'win32':
     _extras['getwinerror'] = interp2app(W_FFIObject.descr_getwinerror)
@@ -770,7 +757,7 @@
                                      cls=W_FFIObject),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
-        buffer      = interp2app(W_FFIObject.descr_buffer),
+        buffer      = ClassAttr(make_buffer),
         callback    = interp2app(W_FFIObject.descr_callback),
         cast        = interp2app(W_FFIObject.descr_cast),
         def_extern  = interp2app(W_FFIObject.descr_def_extern),
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2288,6 +2288,7 @@
     buf = buffer(c)
     assert repr(buf).startswith('<_cffi_backend.buffer object at 0x')
     assert bytes(buf) == b"hi there\x00"
+    assert type(buf) is buffer
     if sys.version_info < (3,):
         assert str(buf) == "hi there\x00"
         assert unicode(buf) == u+"hi there\x00"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -256,6 +256,8 @@
         ffi = _cffi1_backend.FFI()
         a = ffi.new("signed char[]", [5, 6, 7])
         assert ffi.buffer(a)[:] == b'\x05\x06\x07'
+        assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+        assert type(ffi.buffer(a)) is ffi.buffer
 
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1078,7 +1078,8 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI;
+    struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     global_objects = []
diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/genobject.py
@@ -0,0 +1,5 @@
+from pypy.interpreter.generator import GeneratorIterator
+from pypy.module.cpyext.api import build_type_checkers
+
+
+PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -660,17 +660,6 @@
     extension modules."""
     raise NotImplementedError
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_Check(space, ob):
-    """Return true if ob is a generator object; ob must not be NULL."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_CheckExact(space, ob):
-    """Return true if ob's type is PyGen_Type is a generator object; ob must not
-    be NULL."""
-    raise NotImplementedError
-
 @cpython_api([PyFrameObject], PyObject)
 def PyGen_New(space, frame):
     """Create and return a new generator object based on the frame object. A
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -0,0 +1,15 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.genobject import PyGen_Check, PyGen_CheckExact
+
+
+class TestGenObject(BaseApiTest):
+    def test_genobject(self, space):
+        w_geniter = space.appexec([], """():
+            def f():
+                yield 42
+            return f()
+        """)
+        assert PyGen_Check(space, w_geniter)
+        assert PyGen_CheckExact(space, w_geniter)
+        assert not PyGen_Check(space, space.wrap(2))
+        assert not PyGen_CheckExact(space, space.wrap("b"))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1131,6 +1131,7 @@
             b = ffi.buffer(a)
         except NotImplementedError as e:
             py.test.skip(str(e))
+        assert type(b) is ffi.buffer
         content = b[:]
         assert len(content) == len(b) == 2
         if sys.byteorder == 'little':
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -234,6 +234,7 @@
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
     assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+    assert type(ffi.buffer(a)) is ffi.buffer
 
 def test_ffi_from_buffer():
     import array
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -37,6 +37,11 @@
                                       ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
+def test_set_source_no_slashes():
+    ffi = FFI()
+    py.test.raises(ValueError, ffi.set_source, "abc/def", None)
+    py.test.raises(ValueError, ffi.set_source, "abc/def", "C code")
+
 
 def test_type_table_func():
     check_type_table("double sin(double);",
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -602,7 +602,7 @@
 
 if rposix.HAVE_FSTATAT:
     from rpython.rlib.rposix import AT_FDCWD, AT_SYMLINK_NOFOLLOW
-    c_fstatat = rffi.llexternal('fstatat',
+    c_fstatat = rffi.llexternal('fstatat64' if _LINUX else 'fstatat',
         [rffi.INT, rffi.CCHARP, STAT_STRUCT, rffi.INT], rffi.INT,
         compilation_info=compilation_info,
         save_err=rffi.RFFI_SAVE_ERRNO, macro=True)

From pypy.commits at gmail.com  Sun Feb  5 12:01:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 09:01:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: PyCoro_CheckExact()
Message-ID: <58975a53.d3811c0a.be7ac.a213@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89946:1368d86f0ce9
Date: 2017-02-05 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/1368d86f0ce9/

Log:	PyCoro_CheckExact()

diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
--- a/pypy/module/cpyext/genobject.py
+++ b/pypy/module/cpyext/genobject.py
@@ -1,5 +1,7 @@
-from pypy.interpreter.generator import GeneratorIterator
+from pypy.interpreter.generator import GeneratorIterator, Coroutine
 from pypy.module.cpyext.api import build_type_checkers
 
 
 PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
+
+_, PyCoro_CheckExact = build_type_checkers("Coro", Coroutine)
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
--- a/pypy/module/cpyext/test/test_genobject.py
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -1,5 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.genobject import PyGen_Check, PyGen_CheckExact
+from pypy.module.cpyext.genobject import PyCoro_CheckExact
 
 
 class TestGenObject(BaseApiTest):
@@ -11,5 +12,16 @@
         """)
         assert PyGen_Check(space, w_geniter)
         assert PyGen_CheckExact(space, w_geniter)
+        assert not PyCoro_CheckExact(space, w_geniter)
         assert not PyGen_Check(space, space.wrap(2))
         assert not PyGen_CheckExact(space, space.wrap("b"))
+        assert not PyCoro_CheckExact(space, space.wrap([]))
+
+        w_coroutine = space.appexec([], """():
+            async def f():
+                pass
+            return f()
+        """)
+        assert not PyGen_Check(space, w_coroutine)
+        assert not PyGen_CheckExact(space, w_coroutine)
+        assert PyCoro_CheckExact(space, w_coroutine)

From pypy.commits at gmail.com  Sun Feb  5 12:05:59 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 05 Feb 2017 09:05:59 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: kill buffer_w_ex()
Message-ID: <58975b77.1282df0a.bb340.4959@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r89947:d57c072db0e5
Date: 2017-02-05 17:02 +0000
http://bitbucket.org/pypy/pypy/changeset/d57c072db0e5/

Log:	kill buffer_w_ex()

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -25,15 +25,6 @@
                                       reds=['items', 'w_iterator'])
 
 
- at specialize.memo()
-def _does_override_buffer_w(type):
-    return type.buffer_w != W_Root.buffer_w
-
- at specialize.memo()
-def _does_override_buffer_w_ex(type):
-    return type.buffer_w_ex != W_Root.buffer_w_ex
-
-
 class W_Root(object):
     """This is the abstract root class of all wrapped objects that live
     in a 'normal' object space like StdObjSpace."""
@@ -223,15 +214,8 @@
         return None
 
     def buffer_w(self, space, flags):
-        if _does_override_buffer_w_ex(self.__class__):
-            return self.buffer_w_ex(space, flags)[0]
         return self.__buffer_w(space, flags).buffer_w(space, flags)
 
-    def buffer_w_ex(self, space, flags):
-        if _does_override_buffer_w(self.__class__):
-            return self.buffer_w(space, flags), 'B', 1
-        return self.__buffer_w(space, flags).buffer_w_ex(space, flags)
-
     def __buffer_w(self, space, flags):
         if flags & space.BUF_WRITABLE:
             w_impl = space.lookup(self, '__wbuffer__')
@@ -1469,15 +1453,6 @@
             raise oefmt(self.w_TypeError,
                         "'%T' does not support the buffer interface", w_obj)
 
-    def buffer_w_ex(self, w_obj, flags):
-        # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
-        # Returns extra information: (buffer, typecode, itemsize)
-        try:
-            return w_obj.buffer_w_ex(self, flags)
-        except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "'%T' does not support the buffer interface", w_obj)
-
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -189,10 +189,6 @@
         for i in range(len(value)):
             ll_buffer[start + i] = value[i]
 
-    def buffer_w_ex(self, space, flags):
-        buf = self.buffer_w(space, flags)
-        return buf, buf.getformat(), buf.getitemsize()
-
 
 W_ArrayInstance.typedef = TypeDef(
     'ArrayInstance',
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -259,10 +259,6 @@
     def buffer_w(self, space, flags):
         return ArrayBuffer(self, False)
 
-    def buffer_w_ex(self, space, flags):
-        buf = self.buffer_w(space, flags)
-        return buf, buf.getformat(), buf.getitemsize()
-
     def descr_append(self, space, w_x):
         """ append(x)
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -74,17 +74,18 @@
     def setformat(self, value):
         self.format = value
 
-    def buffer_w_ex(self, space, flags):
+    def buffer_w(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf, self.getformat(), self.itemsize
+        return self.buf
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
         if isinstance(w_object, W_MemoryView):
             w_object._check_released(space)
             return W_MemoryView.copy(w_object)
-        return W_MemoryView(*space.buffer_w_ex(w_object, space.BUF_FULL_RO))
+        buf = space.buffer_w(w_object, space.BUF_FULL_RO)
+        return W_MemoryView(buf, buf.getformat(), buf.getitemsize())
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -345,9 +345,6 @@
         return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
                           self.w_size, self.w_strides, self.w_shape)
 
-    def buffer_w_ex(self, space, flags):
-        return self.buffer_w(space, flags), space.str_w(self.w_fmt), space.int_w(self.w_size)
-
 W_MockArray.typedef = TypeDef("MockArray",
     __new__ = interp2app(W_MockArray.descr_new),
 )

From pypy.commits at gmail.com  Sun Feb  5 13:06:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 10:06:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <589769a8.2d88df0a.d95e3.5640@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89949:9ba6ab77fd29
Date: 2017-02-05 19:05 +0100
http://bitbucket.org/pypy/pypy/changeset/9ba6ab77fd29/

Log:	hg merge default

diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -1,6 +1,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref
+from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rthread
 from rpython.rlib.objectmodel import we_are_translated
@@ -317,3 +318,16 @@
     be held.  The thread state must have been reset with a previous call to
     PyThreadState_Clear()."""
 
+ at cpython_api([], lltype.Void)
+def PyOS_AfterFork(space):
+    """Function to update some internal state after a process fork; this should be
+    called in the new process if the Python interpreter will continue to be used.
+    If a new executable is loaded into the new process, this function does not need
+    to be called."""
+    if not space.config.translation.thread:
+        return
+    from pypy.module.thread import os_thread
+    try:
+        os_thread.reinit_threads(space)
+    except OperationError as e:
+        e.write_unraisable(space, "PyOS_AfterFork()")
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1468,14 +1468,6 @@
     one of the strings '<stdin>' or '???'."""
     raise NotImplementedError
 
- at cpython_api([], lltype.Void)
-def PyOS_AfterFork(space):
-    """Function to update some internal state after a process fork; this should be
-    called in the new process if the Python interpreter will continue to be used.
-    If a new executable is loaded into the new process, this function does not need
-    to be called."""
-    raise NotImplementedError
-
 @cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
 def PyOS_CheckStack(space):
     """Return true when the interpreter runs out of stack space.  This is a reliable

From pypy.commits at gmail.com  Sun Feb  5 13:04:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 10:04:14 -0800 (PST)
Subject: [pypy-commit] pypy default: PyOS_AfterFork()
Message-ID: <5897691e.c3161c0a.db8fc.bbf3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89948:1d2ae5c36cef
Date: 2017-02-05 19:03 +0100
http://bitbucket.org/pypy/pypy/changeset/1d2ae5c36cef/

Log:	PyOS_AfterFork()

diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -1,6 +1,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref
+from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rthread
 from rpython.rlib.objectmodel import we_are_translated
@@ -313,3 +314,16 @@
     be held.  The thread state must have been reset with a previous call to
     PyThreadState_Clear()."""
 
+ at cpython_api([], lltype.Void)
+def PyOS_AfterFork(space):
+    """Function to update some internal state after a process fork; this should be
+    called in the new process if the Python interpreter will continue to be used.
+    If a new executable is loaded into the new process, this function does not need
+    to be called."""
+    if not space.config.translation.thread:
+        return
+    from pypy.module.thread import os_thread
+    try:
+        os_thread.reinit_threads(space)
+    except OperationError as e:
+        e.write_unraisable(space, "PyOS_AfterFork()")
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1515,14 +1515,6 @@
     one of the strings '<stdin>' or '???'."""
     raise NotImplementedError
 
- at cpython_api([], lltype.Void)
-def PyOS_AfterFork(space):
-    """Function to update some internal state after a process fork; this should be
-    called in the new process if the Python interpreter will continue to be used.
-    If a new executable is loaded into the new process, this function does not need
-    to be called."""
-    raise NotImplementedError
-
 @cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
 def PyOS_CheckStack(space):
     """Return true when the interpreter runs out of stack space.  This is a reliable

From pypy.commits at gmail.com  Sun Feb  5 13:44:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 10:44:22 -0800 (PST)
Subject: [pypy-commit] pypy default: hg merge dict-move-to-end
Message-ID: <58977286.496f1c0a.e92bb.cbf2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89952:72b034121391
Date: 2017-02-05 19:41 +0100
http://bitbucket.org/pypy/pypy/changeset/72b034121391/

Log:	hg merge dict-move-to-end

	Implement move_to_end(last=True/False) on RPython ordered dicts.
	Minor clean-ups in the ordered dict implementation.

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -14,7 +14,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
     SomeFloat, SomeIterator, SomePBC, SomeNone, SomeTypeOf, s_ImpossibleValue,
     s_Bool, s_None, s_Int, unionof, add_knowntypedata,
-    SomeWeakRef, SomeUnicodeString, SomeByteArray)
+    SomeWeakRef, SomeUnicodeString, SomeByteArray, SomeOrderedDict)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
 from rpython.annotator.binaryop import _dict_can_only_throw_keyerror
@@ -575,6 +575,13 @@
         pair(self, s_key).delitem()
     method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
 
+class __extend__(SomeOrderedDict):
+
+    def method_move_to_end(self, s_key, s_last):
+        assert s_Bool.contains(s_last)
+        pair(self, s_key).delitem()
+    method_move_to_end.can_only_throw = _dict_can_only_throw_keyerror
+
 @op.contains.register(SomeString)
 @op.contains.register(SomeUnicodeString)
 def contains_String(annotator, string, char):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,24 @@
         return
     d.delitem_with_hash(key, h)
 
+def _untranslated_move_to_end(d, key, last):
+    "NOT_RPYTHON"
+    value = d.pop(key)
+    if last:
+        d[key] = value
+    else:
+        items = d.items()
+        d.clear()
+        d[key] = value
+        d.update(items)
+
+ at specialize.call_location()
+def move_to_end(d, key, last=True):
+    if not we_are_translated():
+        _untranslated_move_to_end(d, key, last)
+        return
+    d.move_to_end(key, last)
+
 # ____________________________________________________________
 
 def import_from_mixin(M, special_methods=['__init__', '__del__']):
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config, try_inline)
+    fetch_translated_config, try_inline, move_to_end)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -679,6 +679,16 @@
     assert f(29) == 0
     interpret(f, [27])
 
+def test_rordereddict_move_to_end():
+    d = OrderedDict()
+    d['key1'] = 'val1'
+    d['key2'] = 'val2'
+    d['key3'] = 'val3'
+    move_to_end(d, 'key1')
+    assert d.items() == [('key2', 'val2'), ('key3', 'val3'), ('key1', 'val1')]
+    move_to_end(d, 'key1', last=False)
+    assert d.items() == [('key1', 'val1'), ('key2', 'val2'), ('key3', 'val3')]
+
 def test_import_from_mixin():
     class M:    # old-style
         def f(self):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,15 @@
         hop.exception_is_here()
         hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
 
+    def rtype_method_move_to_end(self, hop):
+        v_dict, v_key, v_last = hop.inputargs(
+            self, self.key_repr, lltype.Bool)
+        if not self.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)  # record that we know about it
+        hop.exception_is_here()
+        hop.gendirectcall(ll_dict_move_to_end, v_dict, v_key, v_last)
+
+
 class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
 
     def rtype_getitem((r_dict, r_key), hop):
@@ -542,16 +551,18 @@
         ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
         assert False
 
-def ll_call_delete_by_entry_index(d, hash, i):
+def ll_call_delete_by_entry_index(d, hash, i, replace_with):
+    # only called from _ll_dict_del, whose @jit.look_inside_iff
+    # condition should control when we get inside here with the jit
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_BYTE)
     elif fun == FUNC_SHORT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_SHORT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_SHORT)
     elif IS_64BIT and fun == FUNC_INT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_INT)
     elif fun == FUNC_LONG:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_LONG)
     else:
         # can't be still FUNC_MUST_REINDEX here
         ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
@@ -805,13 +816,12 @@
     ll_dict_delitem_with_hash(d, key, d.keyhash(key))
 
 def ll_dict_delitem_with_hash(d, key, hash):
-    index = d.lookup_function(d, key, hash, FLAG_DELETE)
+    index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
-    _ll_dict_del(d, index)
+    _ll_dict_del(d, hash, index)
 
- at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, index):
+def _ll_dict_del_entry(d, index):
     d.entries.mark_deleted(index)
     d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
@@ -823,6 +833,11 @@
     if ENTRIES.must_clear_value:
         entry.value = lltype.nullptr(ENTRY.value.TO)
 
+ at jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, hash, index):
+    ll_call_delete_by_entry_index(d, hash, index, DELETED)
+    _ll_dict_del_entry(d, index)
+
     if d.num_live_items == 0:
         # Dict is now empty.  Reset these fields.
         d.num_ever_used_items = 0
@@ -963,7 +978,6 @@
 
 FLAG_LOOKUP = 0
 FLAG_STORE = 1
-FLAG_DELETE = 2
 
 @specialize.memo()
 def _ll_ptr_to_array_of(T):
@@ -985,8 +999,6 @@
     if index >= VALID_OFFSET:
         checkingkey = entries[index - VALID_OFFSET].key
         if direct_compare and checkingkey == key:
-            if store_flag == FLAG_DELETE:
-                indexes[i] = rffi.cast(T, DELETED)
             return index - VALID_OFFSET   # found the entry
         if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
             # correct hash, maybe the key is e.g. a different pointer to
@@ -1000,8 +1012,6 @@
                     # the compare did major nasty stuff to the dict: start over
                     return ll_dict_lookup(d, key, hash, store_flag, T)
             if found:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET
         deletedslot = -1
     elif index == DELETED:
@@ -1030,8 +1040,6 @@
         elif index >= VALID_OFFSET:
             checkingkey = entries[index - VALID_OFFSET].key
             if direct_compare and checkingkey == key:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET   # found the entry
             if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
                 # correct hash, maybe the key is e.g. a different pointer to
@@ -1044,8 +1052,6 @@
                         # the compare did major nasty stuff to the dict: start over
                         return ll_dict_lookup(d, key, hash, store_flag, T)
                 if found:
-                    if store_flag == FLAG_DELETE:
-                        indexes[i] = rffi.cast(T, DELETED)
                     return index - VALID_OFFSET
         elif deletedslot == -1:
             deletedslot = intmask(i)
@@ -1066,7 +1072,11 @@
         perturb >>= PERTURB_SHIFT
     indexes[i] = rffi.cast(T, index + VALID_OFFSET)
 
-def ll_dict_delete_by_entry_index(d, hash, locate_index, T):
+# the following function is only called from _ll_dict_del, whose
+# @jit.look_inside_iff condition should control when we get inside
+# here with the jit
+ at jit.unroll_safe
+def ll_dict_delete_by_entry_index(d, hash, locate_index, replace_with, T):
     # Another simplified version of ll_dict_lookup() which locates a
     # hashtable entry with the given 'index' stored in it, and deletes it.
     # This *should* be safe against evil user-level __eq__/__hash__
@@ -1083,7 +1093,7 @@
         i = (i << 2) + i + perturb + 1
         i = i & mask
         perturb >>= PERTURB_SHIFT
-    indexes[i] = rffi.cast(T, DELETED)
+    indexes[i] = rffi.cast(T, replace_with)
 
 # ____________________________________________________________
 #
@@ -1253,18 +1263,8 @@
     if hasattr(DICT, 'fnkeyhash'):
         newdict.fnkeyhash = dict.fnkeyhash
 
-    i = 0
-    while i < newdict.num_ever_used_items:
-        d_entry = newdict.entries[i]
-        entry = dict.entries[i]
-        ENTRY = lltype.typeOf(newdict.entries).TO.OF
-        d_entry.key = entry.key
-        if hasattr(ENTRY, 'f_valid'):
-            d_entry.f_valid = entry.f_valid
-        d_entry.value = entry.value
-        if hasattr(ENTRY, 'f_hash'):
-            d_entry.f_hash = entry.f_hash
-        i += 1
+    rgc.ll_arraycopy(dict.entries, newdict.entries, 0, 0,
+                     newdict.num_ever_used_items)
 
     ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
     return newdict
@@ -1390,8 +1390,6 @@
             break
         dic.num_ever_used_items -= 1
 
-    # we must remove the precise entry in the hashtable that points to 'i'
-    ll_call_delete_by_entry_index(dic, entries.hash(i), i)
     return i
 
 def ll_dict_popitem(ELEM, dic):
@@ -1400,21 +1398,139 @@
     r = lltype.malloc(ELEM.TO)
     r.item0 = recast(ELEM.TO.item0, entry.key)
     r.item1 = recast(ELEM.TO.item1, entry.value)
-    _ll_dict_del(dic, i)
+    _ll_dict_del(dic, dic.entries.hash(i), i)
     return r
 
 def ll_dict_pop(dic, key):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value
 
 def ll_dict_pop_default(dic, key, dfl):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         return dfl
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value
+
+def ll_dict_move_to_end(d, key, last):
+    if last:
+        ll_dict_move_to_last(d, key)
+    else:
+        ll_dict_move_to_first(d, key)
+
+def ll_dict_move_to_last(d, key):
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index < 0:
+        raise KeyError
+
+    if old_index == d.num_ever_used_items - 1:
+        return
+
+    # remove the entry at the old position
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    _ll_dict_del_entry(d, old_index)
+
+    # note a corner case: it is possible that 'replace_with' is just too
+    # large to fit in the type T used so far for the index.  But in that
+    # case, the list 'd.entries' is full, and the following call to
+    # _ll_dict_setitem_lookup_done() will necessarily reindex the dict.
+    # So in that case, this value of 'replace_with' should be ignored.
+    ll_call_delete_by_entry_index(d, hash, old_index,
+            replace_with = VALID_OFFSET + d.num_ever_used_items)
+    _ll_dict_setitem_lookup_done(d, key, value, hash, -1)
+
+def ll_dict_move_to_first(d, key):
+    # In this function, we might do a bit more than the strict minimum
+    # of walks over parts of the array, trying to keep the code at least
+    # semi-reasonable, while the goal is still amortized constant-time
+    # over many calls.
+
+    # Call ll_dict_remove_deleted_items() first if there are too many
+    # deleted items.  Not a perfect solution, because lookup_function()
+    # might do random things with the dict and create many new deleted
+    # items.  Still, should be fine, because nothing crucially depends
+    # on this: the goal is to avoid the dictionary's list growing
+    # forever.
+    if d.num_live_items < len(d.entries) // 2 - 16:
+        ll_dict_remove_deleted_items(d)
+
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index <= 0:
+        if old_index < 0:
+            raise KeyError
+        else:
+            return
+
+    # the goal of the following is to set 'idst' to the number of
+    # deleted entries at the beginning, ensuring 'idst > 0'
+    must_reindex = False
+    if d.entries.valid(0):
+        # the first entry is valid, so we need to make room before.
+        new_allocated = _overallocate_entries_len(d.num_ever_used_items)
+        idst = ((new_allocated - d.num_ever_used_items) * 3) // 4
+        ll_assert(idst > 0, "overallocate did not do enough")
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+        rgc.ll_arraycopy(d.entries, newitems, 0, idst, d.num_ever_used_items)
+        d.entries = newitems
+        i = 0
+        while i < idst:
+            d.entries.mark_deleted(i)
+            i += 1
+        d.num_ever_used_items += idst
+        old_index += idst
+        must_reindex = True
+        idst -= 1
+    else:
+        idst = d.lookup_function_no >> FUNC_SHIFT
+        # All entries in range(0, idst) are deleted.  Check if more are
+        while not d.entries.valid(idst):
+            idst += 1
+        if idst == old_index:
+            d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                    (old_index << FUNC_SHIFT))
+            return
+        idst -= 1
+        d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                (idst << FUNC_SHIFT))
+
+    # remove the entry at the old position
+    ll_assert(d.entries.valid(old_index),
+              "ll_dict_move_to_first: lost old_index")
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    if hasattr(ENTRY, 'f_hash'):
+        ll_assert(old_entry.f_hash == hash,
+                  "ll_dict_move_to_first: bad hash")
+    _ll_dict_del_entry(d, old_index)
+
+    # put the entry at its new position
+    ll_assert(not d.entries.valid(idst),
+              "ll_dict_move_to_first: overwriting idst")
+    new_entry = d.entries[idst]
+    new_entry.key = key
+    new_entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        new_entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        new_entry.f_valid = True
+    d.num_live_items += 1
+
+    # fix the index
+    if must_reindex:
+        ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+    else:
+        ll_call_delete_by_entry_index(d, hash, old_index,
+                replace_with = VALID_OFFSET + idst)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1188,6 +1188,12 @@
         assert not self.ll_contains(self.l_dict, ll_key)
         self.removed_keys.append(key)
 
+    def move_to_end(self, key, last=True):
+        "For test_rordereddict"
+
+    def move_to_first(self, key):
+        self.move_to_end(key, last=False)
+
     def copydict(self):
         self.l_dict = self.ll_copy(self.l_dict)
         assert self.ll_len(self.l_dict) == len(self.reference)
@@ -1250,6 +1256,15 @@
         return builds(Action,
             just('delitem'), tuples(sampled_from(self.space.reference)))
 
+    def st_move_to_end(self):
+        return builds(Action,
+            just('move_to_end'), tuples(sampled_from(self.space.reference)))
+
+    def st_move_to_first(self):
+        return builds(Action,
+            just('move_to_first'),
+                tuples(sampled_from(self.space.reference)))
+
     def steps(self):
         if not self.space:
             return builds(Action, just('setup'), tuples(st_keys, st_values))
@@ -1258,7 +1273,8 @@
         if self.space.reference:
             return (
                 self.st_setitem() | sampled_from(global_actions) |
-                self.st_updateitem() | self.st_delitem())
+                self.st_updateitem() | self.st_delitem() |
+                self.st_move_to_end() | self.st_move_to_first())
         else:
             return (self.st_setitem() | sampled_from(global_actions))
 
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -1,7 +1,8 @@
 import py
+import random
 from collections import OrderedDict
 
-from hypothesis import settings
+from hypothesis import settings, given, strategies
 from hypothesis.stateful import run_state_machine_as_test
 
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -145,14 +146,18 @@
         ll_d = rordereddict.ll_newdict(DICT)
         rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
         rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
-        ITER = rordereddict.get_ll_dictiter(lltype.Ptr(DICT))
+        assert [hlstr(entry.key) for entry in self._ll_iter(ll_d)] == ["k", "j"]
+
+    def _ll_iter(self, ll_d):
+        ITER = rordereddict.get_ll_dictiter(lltype.typeOf(ll_d))
         ll_iter = rordereddict.ll_dictiter(ITER, ll_d)
         ll_dictnext = rordereddict._ll_dictnext
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "k"
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "j"
-        py.test.raises(StopIteration, ll_dictnext, ll_iter)
+        while True:
+            try:
+                num = ll_dictnext(ll_iter)
+            except StopIteration:
+                break
+            yield ll_d.entries[num]
 
     def test_popitem(self):
         DICT = self._get_str_dict()
@@ -337,6 +342,31 @@
             num_nonfrees += (got > 0)
         assert d.resize_counter <= idx.getlength() * 2 - num_nonfrees * 3
 
+    @given(strategies.lists(strategies.integers(min_value=1, max_value=5)))
+    def test_direct_move_to_end(self, lst):
+        DICT = self._get_int_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, 1, 11)
+        rordereddict.ll_dict_setitem(ll_d, 2, 22)
+        def content():
+            return [(entry.key, entry.value) for entry in self._ll_iter(ll_d)]
+        for case in lst:
+            if case == 1:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, True)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 2:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, True)
+                assert content() == [(1, 11), (2, 22)]
+            elif case == 3:
+                py.test.raises(KeyError, rordereddict.ll_dict_move_to_end,
+                                                 ll_d, 3, True)
+            elif case == 4:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, False)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 5:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, False)
+                assert content() == [(1, 11), (2, 22)]
+
 
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
@@ -369,10 +399,29 @@
         res = self.interpret(func, [5])
         assert res == 6
 
+    def test_move_to_end(self):
+        def func():
+            d1 = OrderedDict()
+            d1['key1'] = 'value1'
+            d1['key2'] = 'value2'
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key1')
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key2')
+                assert d1.keys() == ['key1', 'key2']
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key2', last=False)
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key1', last=False)
+                assert d1.keys() == ['key1', 'key2']
+        func()
+        self.interpret(func, [])
+
 
 class ODictSpace(MappingSpace):
     MappingRepr = rodct.OrderedDictRepr
     new_reference = OrderedDict
+    moved_around = False
     ll_getitem = staticmethod(rodct.ll_dict_getitem)
     ll_setitem = staticmethod(rodct.ll_dict_setitem)
     ll_delitem = staticmethod(rodct.ll_dict_delitem)
@@ -417,9 +466,23 @@
     def removeindex(self):
         # remove the index, as done during translation for prebuilt dicts
         # (but cannot be done if we already removed a key)
-        if not self.removed_keys:
+        if not self.removed_keys and not self.moved_around:
             rodct.ll_no_initial_index(self.l_dict)
 
+    def move_to_end(self, key, last=True):
+        ll_key = self.ll_key(key)
+        rodct.ll_dict_move_to_end(self.l_dict, ll_key, last)
+        value = self.reference.pop(key)
+        if last:
+            self.reference[key] = value
+        else:
+            items = self.reference.items()
+            self.reference.clear()
+            self.reference[key] = value
+            self.reference.update(items)
+        # prevent ll_no_initial_index()
+        self.moved_around = True
+
     def fullcheck(self):
         # overridden to also check key order
         assert self.ll_len(self.l_dict) == len(self.reference)

From pypy.commits at gmail.com  Sun Feb  5 13:44:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 10:44:20 -0800 (PST)
Subject: [pypy-commit] pypy dict-move-to-end: ready to merge
Message-ID: <58977284.2da9df0a.7f654.66b4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: dict-move-to-end
Changeset: r89951:8a180d361371
Date: 2017-02-05 19:40 +0100
http://bitbucket.org/pypy/pypy/changeset/8a180d361371/

Log:	ready to merge


From pypy.commits at gmail.com  Sun Feb  5 13:44:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 10:44:18 -0800 (PST)
Subject: [pypy-commit] pypy vendor/stdlib-3.5: Import stdlib from CPython
 3.5.3 (changeset 'v3.5.3')
Message-ID: <58977282.862f1c0a.af111.cbf3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: vendor/stdlib-3.5
Changeset: r89950:a9167602e69c
Date: 2017-02-05 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/a9167602e69c/

Log:	Import stdlib from CPython 3.5.3 (changeset 'v3.5.3')

diff too long, truncating to 2000 out of 24911 lines

diff --git a/lib-python/3/_collections_abc.py b/lib-python/3/_collections_abc.py
--- a/lib-python/3/_collections_abc.py
+++ b/lib-python/3/_collections_abc.py
@@ -29,8 +29,8 @@
 # so that they will pass tests like:
 #       it = iter(somebytearray)
 #       assert isinstance(it, Iterable)
-# Note:  in other implementations, these types many not be distinct
-# and they make have their own implementation specific types that
+# Note:  in other implementations, these types might not be distinct
+# and they may have their own implementation specific types that
 # are not included on this list.
 bytes_iterator = type(iter(b''))
 bytearray_iterator = type(iter(bytearray()))
@@ -41,6 +41,7 @@
 list_iterator = type(iter([]))
 list_reverseiterator = type(iter(reversed([])))
 range_iterator = type(iter(range(0)))
+longrange_iterator = type(iter(range(1 << 1000)))
 set_iterator = type(iter(set()))
 str_iterator = type(iter(""))
 tuple_iterator = type(iter(()))
@@ -234,6 +235,7 @@
 Iterator.register(list_iterator)
 Iterator.register(list_reverseiterator)
 Iterator.register(range_iterator)
+Iterator.register(longrange_iterator)
 Iterator.register(set_iterator)
 Iterator.register(str_iterator)
 Iterator.register(tuple_iterator)
diff --git a/lib-python/3/_pydecimal.py b/lib-python/3/_pydecimal.py
--- a/lib-python/3/_pydecimal.py
+++ b/lib-python/3/_pydecimal.py
@@ -1068,12 +1068,11 @@
         return sign + intpart + fracpart + exp
 
     def to_eng_string(self, context=None):
-        """Convert to engineering-type string.
-
-        Engineering notation has an exponent which is a multiple of 3, so there
-        are up to 3 digits left of the decimal place.
-
-        Same rules for when in exponential and when as a value as in __str__.
+        """Convert to a string, using engineering notation if an exponent is needed.
+
+        Engineering notation has an exponent which is a multiple of 3.  This
+        can leave up to 3 digits to the left of the decimal place and may
+        require the addition of either one or two trailing zeros.
         """
         return self.__str__(eng=True, context=context)
 
@@ -4107,7 +4106,7 @@
         >>> context.create_decimal_from_float(3.1415926535897932)
         Traceback (most recent call last):
             ...
-        decimal.Inexact
+        decimal.Inexact: None
 
         """
         d = Decimal.from_float(f)       # An exact conversion
@@ -5502,9 +5501,29 @@
             return r
 
     def to_eng_string(self, a):
-        """Converts a number to a string, using scientific notation.
+        """Convert to a string, using engineering notation if an exponent is needed.
+
+        Engineering notation has an exponent which is a multiple of 3.  This
+        can leave up to 3 digits to the left of the decimal place and may
+        require the addition of either one or two trailing zeros.
 
         The operation is not affected by the context.
+
+        >>> ExtendedContext.to_eng_string(Decimal('123E+1'))
+        '1.23E+3'
+        >>> ExtendedContext.to_eng_string(Decimal('123E+3'))
+        '123E+3'
+        >>> ExtendedContext.to_eng_string(Decimal('123E-10'))
+        '12.3E-9'
+        >>> ExtendedContext.to_eng_string(Decimal('-123E-12'))
+        '-123E-12'
+        >>> ExtendedContext.to_eng_string(Decimal('7E-7'))
+        '700E-9'
+        >>> ExtendedContext.to_eng_string(Decimal('7E+1'))
+        '70'
+        >>> ExtendedContext.to_eng_string(Decimal('0E+1'))
+        '0.00E+3'
+
         """
         a = _convert_other(a, raiseit=True)
         return a.to_eng_string(context=self)
diff --git a/lib-python/3/_pyio.py b/lib-python/3/_pyio.py
--- a/lib-python/3/_pyio.py
+++ b/lib-python/3/_pyio.py
@@ -276,7 +276,7 @@
 try:
     UnsupportedOperation = io.UnsupportedOperation
 except AttributeError:
-    class UnsupportedOperation(ValueError, OSError):
+    class UnsupportedOperation(OSError, ValueError):
         pass
 
 
diff --git a/lib-python/3/antigravity.py b/lib-python/3/antigravity.py
--- a/lib-python/3/antigravity.py
+++ b/lib-python/3/antigravity.py
@@ -2,7 +2,7 @@
 import webbrowser
 import hashlib
 
-webbrowser.open("http://xkcd.com/353/")
+webbrowser.open("https://xkcd.com/353/")
 
 def geohash(latitude, longitude, datedow):
     '''Compute geohash() using the Munroe algorithm.
diff --git a/lib-python/3/asyncio/base_events.py b/lib-python/3/asyncio/base_events.py
--- a/lib-python/3/asyncio/base_events.py
+++ b/lib-python/3/asyncio/base_events.py
@@ -13,7 +13,6 @@
 to modify the meaning of the API call itself.
 """
 
-
 import collections
 import concurrent.futures
 import heapq
@@ -28,6 +27,7 @@
 import traceback
 import sys
 import warnings
+import weakref
 
 from . import compat
 from . import coroutines
@@ -41,9 +41,6 @@
 __all__ = ['BaseEventLoop']
 
 
-# Argument for default thread pool executor creation.
-_MAX_WORKERS = 5
-
 # Minimum number of _scheduled timer handles before cleanup of
 # cancelled handles is performed.
 _MIN_SCHEDULED_TIMER_HANDLES = 100
@@ -76,12 +73,29 @@
         return repr(fd)
 
 
-# Linux's sock.type is a bitmask that can include extra info about socket.
-_SOCKET_TYPE_MASK = 0
-if hasattr(socket, 'SOCK_NONBLOCK'):
-    _SOCKET_TYPE_MASK |= socket.SOCK_NONBLOCK
-if hasattr(socket, 'SOCK_CLOEXEC'):
-    _SOCKET_TYPE_MASK |= socket.SOCK_CLOEXEC
+def _set_reuseport(sock):
+    if not hasattr(socket, 'SO_REUSEPORT'):
+        raise ValueError('reuse_port not supported by socket module')
+    else:
+        try:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        except OSError:
+            raise ValueError('reuse_port not supported by socket module, '
+                             'SO_REUSEPORT defined but not implemented.')
+
+
+def _is_stream_socket(sock):
+    # Linux's socket.type is a bitmask that can include extra info
+    # about socket, therefore we can't do simple
+    # `sock_type == socket.SOCK_STREAM`.
+    return (sock.type & socket.SOCK_STREAM) == socket.SOCK_STREAM
+
+
+def _is_dgram_socket(sock):
+    # Linux's socket.type is a bitmask that can include extra info
+    # about socket, therefore we can't do simple
+    # `sock_type == socket.SOCK_DGRAM`.
+    return (sock.type & socket.SOCK_DGRAM) == socket.SOCK_DGRAM
 
 
 def _ipaddr_info(host, port, family, type, proto):
@@ -94,8 +108,12 @@
             host is None:
         return None
 
-    type &= ~_SOCKET_TYPE_MASK
     if type == socket.SOCK_STREAM:
+        # Linux only:
+        #    getaddrinfo() can raise when socket.type is a bit mask.
+        #    So if socket.type is a bit mask of SOCK_STREAM, and say
+        #    SOCK_NONBLOCK, we simply return None, which will trigger
+        #    a call to getaddrinfo() letting it process this request.
         proto = socket.IPPROTO_TCP
     elif type == socket.SOCK_DGRAM:
         proto = socket.IPPROTO_UDP
@@ -104,27 +122,21 @@
 
     if port is None:
         port = 0
-    elif isinstance(port, bytes):
-        if port == b'':
-            port = 0
-        else:
-            try:
-                port = int(port)
-            except ValueError:
-                # Might be a service name like b"http".
-                port = socket.getservbyname(port.decode('ascii'))
-    elif isinstance(port, str):
-        if port == '':
-            port = 0
-        else:
-            try:
-                port = int(port)
-            except ValueError:
-                # Might be a service name like "http".
-                port = socket.getservbyname(port)
+    elif isinstance(port, bytes) and port == b'':
+        port = 0
+    elif isinstance(port, str) and port == '':
+        port = 0
+    else:
+        # If port's a service name like "http", don't skip getaddrinfo.
+        try:
+            port = int(port)
+        except (TypeError, ValueError):
+            return None
 
     if family == socket.AF_UNSPEC:
-        afs = [socket.AF_INET, socket.AF_INET6]
+        afs = [socket.AF_INET]
+        if hasattr(socket, 'AF_INET6'):
+            afs.append(socket.AF_INET6)
     else:
         afs = [family]
 
@@ -242,6 +254,17 @@
         self._task_factory = None
         self._coroutine_wrapper_set = False
 
+        if hasattr(sys, 'get_asyncgen_hooks'):
+            # Python >= 3.6
+            # A weak set of all asynchronous generators that are
+            # being iterated by the loop.
+            self._asyncgens = weakref.WeakSet()
+        else:
+            self._asyncgens = None
+
+        # Set to True when `loop.shutdown_asyncgens` is called.
+        self._asyncgens_shutdown_called = False
+
     def __repr__(self):
         return ('<%s running=%s closed=%s debug=%s>'
                 % (self.__class__.__name__, self.is_running(),
@@ -333,14 +356,67 @@
         if self._closed:
             raise RuntimeError('Event loop is closed')
 
+    def _asyncgen_finalizer_hook(self, agen):
+        self._asyncgens.discard(agen)
+        if not self.is_closed():
+            self.create_task(agen.aclose())
+            # Wake up the loop if the finalizer was called from
+            # a different thread.
+            self._write_to_self()
+
+    def _asyncgen_firstiter_hook(self, agen):
+        if self._asyncgens_shutdown_called:
+            warnings.warn(
+                "asynchronous generator {!r} was scheduled after "
+                "loop.shutdown_asyncgens() call".format(agen),
+                ResourceWarning, source=self)
+
+        self._asyncgens.add(agen)
+
+    @coroutine
+    def shutdown_asyncgens(self):
+        """Shutdown all active asynchronous generators."""
+        self._asyncgens_shutdown_called = True
+
+        if self._asyncgens is None or not len(self._asyncgens):
+            # If Python version is <3.6 or we don't have any asynchronous
+            # generators alive.
+            return
+
+        closing_agens = list(self._asyncgens)
+        self._asyncgens.clear()
+
+        shutdown_coro = tasks.gather(
+            *[ag.aclose() for ag in closing_agens],
+            return_exceptions=True,
+            loop=self)
+
+        results = yield from shutdown_coro
+        for result, agen in zip(results, closing_agens):
+            if isinstance(result, Exception):
+                self.call_exception_handler({
+                    'message': 'an error occurred during closing of '
+                               'asynchronous generator {!r}'.format(agen),
+                    'exception': result,
+                    'asyncgen': agen
+                })
+
     def run_forever(self):
         """Run until stop() is called."""
         self._check_closed()
         if self.is_running():
-            raise RuntimeError('Event loop is running.')
+            raise RuntimeError('This event loop is already running')
+        if events._get_running_loop() is not None:
+            raise RuntimeError(
+                'Cannot run the event loop while another loop is running')
         self._set_coroutine_wrapper(self._debug)
         self._thread_id = threading.get_ident()
+        if self._asyncgens is not None:
+            old_agen_hooks = sys.get_asyncgen_hooks()
+            sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook,
+                                   finalizer=self._asyncgen_finalizer_hook)
         try:
+            events._set_running_loop(self)
             while True:
                 self._run_once()
                 if self._stopping:
@@ -348,7 +424,10 @@
         finally:
             self._stopping = False
             self._thread_id = None
+            events._set_running_loop(None)
             self._set_coroutine_wrapper(False)
+            if self._asyncgens is not None:
+                sys.set_asyncgen_hooks(*old_agen_hooks)
 
     def run_until_complete(self, future):
         """Run until the Future is done.
@@ -363,7 +442,7 @@
         """
         self._check_closed()
 
-        new_task = not isinstance(future, futures.Future)
+        new_task = not futures.isfuture(future)
         future = tasks.ensure_future(future, loop=self)
         if new_task:
             # An exception is raised if the future didn't complete, so there
@@ -469,12 +548,10 @@
 
         Absolute time corresponds to the event loop's time() method.
         """
-        if (coroutines.iscoroutine(callback)
-        or coroutines.iscoroutinefunction(callback)):
-            raise TypeError("coroutines cannot be used with call_at()")
         self._check_closed()
         if self._debug:
             self._check_thread()
+            self._check_callback(callback, 'call_at')
         timer = events.TimerHandle(when, callback, args, self)
         if timer._source_traceback:
             del timer._source_traceback[-1]
@@ -492,18 +569,27 @@
         Any positional arguments after the callback will be passed to
         the callback when it is called.
         """
+        self._check_closed()
         if self._debug:
             self._check_thread()
+            self._check_callback(callback, 'call_soon')
         handle = self._call_soon(callback, args)
         if handle._source_traceback:
             del handle._source_traceback[-1]
         return handle
 
+    def _check_callback(self, callback, method):
+        if (coroutines.iscoroutine(callback) or
+                coroutines.iscoroutinefunction(callback)):
+            raise TypeError(
+                "coroutines cannot be used with {}()".format(method))
+        if not callable(callback):
+            raise TypeError(
+                'a callable object was expected by {}(), got {!r}'.format(
+                    method, callback))
+
+
     def _call_soon(self, callback, args):
-        if (coroutines.iscoroutine(callback)
-        or coroutines.iscoroutinefunction(callback)):
-            raise TypeError("coroutines cannot be used with call_soon()")
-        self._check_closed()
         handle = events.Handle(callback, args, self)
         if handle._source_traceback:
             del handle._source_traceback[-1]
@@ -529,6 +615,9 @@
 
     def call_soon_threadsafe(self, callback, *args):
         """Like call_soon(), but thread-safe."""
+        self._check_closed()
+        if self._debug:
+            self._check_callback(callback, 'call_soon_threadsafe')
         handle = self._call_soon(callback, args)
         if handle._source_traceback:
             del handle._source_traceback[-1]
@@ -536,22 +625,13 @@
         return handle
 
     def run_in_executor(self, executor, func, *args):
-        if (coroutines.iscoroutine(func)
-        or coroutines.iscoroutinefunction(func)):
-            raise TypeError("coroutines cannot be used with run_in_executor()")
         self._check_closed()
-        if isinstance(func, events.Handle):
-            assert not args
-            assert not isinstance(func, events.TimerHandle)
-            if func._cancelled:
-                f = self.create_future()
-                f.set_result(None)
-                return f
-            func, args = func._callback, func._args
+        if self._debug:
+            self._check_callback(func, 'run_in_executor')
         if executor is None:
             executor = self._default_executor
             if executor is None:
-                executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS)
+                executor = concurrent.futures.ThreadPoolExecutor()
                 self._default_executor = executor
         return futures.wrap_future(executor.submit(func, *args), loop=self)
 
@@ -703,11 +783,19 @@
                     raise OSError('Multiple exceptions: {}'.format(
                         ', '.join(str(exc) for exc in exceptions)))
 
-        elif sock is None:
-            raise ValueError(
-                'host and port was not specified and no sock specified')
-
-        sock.setblocking(False)
+        else:
+            if sock is None:
+                raise ValueError(
+                    'host and port was not specified and no sock specified')
+            if not _is_stream_socket(sock):
+                # We allow AF_INET, AF_INET6, AF_UNIX as long as they
+                # are SOCK_STREAM.
+                # We support passing AF_UNIX sockets even though we have
+                # a dedicated API for that: create_unix_connection.
+                # Disallowing AF_UNIX in this method, breaks backwards
+                # compatibility.
+                raise ValueError(
+                    'A Stream Socket was expected, got {!r}'.format(sock))
 
         transport, protocol = yield from self._create_connection_transport(
             sock, protocol_factory, ssl, server_hostname)
@@ -721,14 +809,17 @@
 
     @coroutine
     def _create_connection_transport(self, sock, protocol_factory, ssl,
-                                     server_hostname):
+                                     server_hostname, server_side=False):
+
+        sock.setblocking(False)
+
         protocol = protocol_factory()
         waiter = self.create_future()
         if ssl:
             sslcontext = None if isinstance(ssl, bool) else ssl
             transport = self._make_ssl_transport(
                 sock, protocol, sslcontext, waiter,
-                server_side=False, server_hostname=server_hostname)
+                server_side=server_side, server_hostname=server_hostname)
         else:
             transport = self._make_socket_transport(sock, protocol, waiter)
 
@@ -748,6 +839,9 @@
                                  allow_broadcast=None, sock=None):
         """Create datagram connection."""
         if sock is not None:
+            if not _is_dgram_socket(sock):
+                raise ValueError(
+                    'A UDP Socket was expected, got {!r}'.format(sock))
             if (local_addr or remote_addr or
                     family or proto or flags or
                     reuse_address or reuse_port or allow_broadcast):
@@ -813,12 +907,7 @@
                         sock.setsockopt(
                             socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                     if reuse_port:
-                        if not hasattr(socket, 'SO_REUSEPORT'):
-                            raise ValueError(
-                                'reuse_port not supported by socket module')
-                        else:
-                            sock.setsockopt(
-                                socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+                        _set_reuseport(sock)
                     if allow_broadcast:
                         sock.setsockopt(
                             socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
@@ -941,12 +1030,7 @@
                         sock.setsockopt(
                             socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
                     if reuse_port:
-                        if not hasattr(socket, 'SO_REUSEPORT'):
-                            raise ValueError(
-                                'reuse_port not supported by socket module')
-                        else:
-                            sock.setsockopt(
-                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
+                        _set_reuseport(sock)
                     # Disable IPv4/IPv6 dual stack support (enabled by
                     # default on Linux) which makes a single socket
                     # listen on both address families.
@@ -968,18 +1052,44 @@
         else:
             if sock is None:
                 raise ValueError('Neither host/port nor sock were specified')
+            if not _is_stream_socket(sock):
+                raise ValueError(
+                    'A Stream Socket was expected, got {!r}'.format(sock))
             sockets = [sock]
 
         server = Server(self, sockets)
         for sock in sockets:
             sock.listen(backlog)
             sock.setblocking(False)
-            self._start_serving(protocol_factory, sock, ssl, server)
+            self._start_serving(protocol_factory, sock, ssl, server, backlog)
         if self._debug:
             logger.info("%r is serving", server)
         return server
 
     @coroutine
+    def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None):
+        """Handle an accepted connection.
+
+        This is used by servers that accept connections outside of
+        asyncio but that use asyncio to handle connections.
+
+        This method is a coroutine.  When completed, the coroutine
+        returns a (transport, protocol) pair.
+        """
+        if not _is_stream_socket(sock):
+            raise ValueError(
+                'A Stream Socket was expected, got {!r}'.format(sock))
+
+        transport, protocol = yield from self._create_connection_transport(
+            sock, protocol_factory, ssl, '', server_side=True)
+        if self._debug:
+            # Get the socket from the transport because SSL transport closes
+            # the old socket and creates a new SSL socket
+            sock = transport.get_extra_info('socket')
+            logger.debug("%r handled: (%r, %r)", sock, transport, protocol)
+        return transport, protocol
+
+    @coroutine
     def connect_read_pipe(self, protocol_factory, pipe):
         protocol = protocol_factory()
         waiter = self.create_future()
@@ -1048,7 +1158,7 @@
         transport = yield from self._make_subprocess_transport(
             protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
         if self._debug:
-            logger.info('%s: %r' % (debug_log, transport))
+            logger.info('%s: %r', debug_log, transport)
         return transport, protocol
 
     @coroutine
@@ -1078,7 +1188,7 @@
             protocol, popen_args, False, stdin, stdout, stderr,
             bufsize, **kwargs)
         if self._debug:
-            logger.info('%s: %r' % (debug_log, transport))
+            logger.info('%s: %r', debug_log, transport)
         return transport, protocol
 
     def get_exception_handler(self):
@@ -1158,7 +1268,9 @@
         - 'handle' (optional): Handle instance;
         - 'protocol' (optional): Protocol instance;
         - 'transport' (optional): Transport instance;
-        - 'socket' (optional): Socket instance.
+        - 'socket' (optional): Socket instance;
+        - 'asyncgen' (optional): Asynchronous generator that caused
+                                 the exception.
 
         New keys maybe introduced in the future.
 
diff --git a/lib-python/3/asyncio/base_subprocess.py b/lib-python/3/asyncio/base_subprocess.py
--- a/lib-python/3/asyncio/base_subprocess.py
+++ b/lib-python/3/asyncio/base_subprocess.py
@@ -3,7 +3,6 @@
 import warnings
 
 from . import compat
-from . import futures
 from . import protocols
 from . import transports
 from .coroutines import coroutine
@@ -87,6 +86,12 @@
     def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
         raise NotImplementedError
 
+    def set_protocol(self, protocol):
+        self._protocol = protocol
+
+    def get_protocol(self):
+        return self._protocol
+
     def is_closing(self):
         return self._closed
 
diff --git a/lib-python/3/asyncio/coroutines.py b/lib-python/3/asyncio/coroutines.py
--- a/lib-python/3/asyncio/coroutines.py
+++ b/lib-python/3/asyncio/coroutines.py
@@ -33,12 +33,16 @@
 
 try:
     _types_coroutine = types.coroutine
+    _types_CoroutineType = types.CoroutineType
 except AttributeError:
+    # Python 3.4
     _types_coroutine = None
+    _types_CoroutineType = None
 
 try:
     _inspect_iscoroutinefunction = inspect.iscoroutinefunction
 except AttributeError:
+    # Python 3.4
     _inspect_iscoroutinefunction = lambda func: False
 
 try:
@@ -120,8 +124,8 @@
         def send(self, value):
             return self.gen.send(value)
 
-    def throw(self, exc):
-        return self.gen.throw(exc)
+    def throw(self, type, value=None, traceback=None):
+        return self.gen.throw(type, value, traceback)
 
     def close(self):
         return self.gen.close()
@@ -204,8 +208,8 @@
         @functools.wraps(func)
         def coro(*args, **kw):
             res = func(*args, **kw)
-            if isinstance(res, futures.Future) or inspect.isgenerator(res) or \
-                    isinstance(res, CoroWrapper):
+            if (futures.isfuture(res) or inspect.isgenerator(res) or
+                isinstance(res, CoroWrapper)):
                 res = yield from res
             elif _AwaitableABC is not None:
                 # If 'func' returns an Awaitable (new in 3.5) we
@@ -238,19 +242,27 @@
             w.__qualname__ = getattr(func, '__qualname__', None)
             return w
 
-    wrapper._is_coroutine = True  # For iscoroutinefunction().
+    wrapper._is_coroutine = _is_coroutine  # For iscoroutinefunction().
     return wrapper
 
 
+# A marker for iscoroutinefunction.
+_is_coroutine = object()
+
+
 def iscoroutinefunction(func):
     """Return True if func is a decorated coroutine function."""
-    return (getattr(func, '_is_coroutine', False) or
+    return (getattr(func, '_is_coroutine', None) is _is_coroutine or
             _inspect_iscoroutinefunction(func))
 
 
 _COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)
 if _CoroutineABC is not None:
     _COROUTINE_TYPES += (_CoroutineABC,)
+if _types_CoroutineType is not None:
+    # Prioritize native coroutine check to speed-up
+    # asyncio.iscoroutine.
+    _COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES
 
 
 def iscoroutine(obj):
@@ -261,6 +273,29 @@
 def _format_coroutine(coro):
     assert iscoroutine(coro)
 
+    if not hasattr(coro, 'cr_code') and not hasattr(coro, 'gi_code'):
+        # Most likely a built-in type or a Cython coroutine.
+
+        # Built-in types might not have __qualname__ or __name__.
+        coro_name = getattr(
+            coro, '__qualname__',
+            getattr(coro, '__name__', type(coro).__name__))
+        coro_name = '{}()'.format(coro_name)
+
+        running = False
+        try:
+            running = coro.cr_running
+        except AttributeError:
+            try:
+                running = coro.gi_running
+            except AttributeError:
+                pass
+
+        if running:
+            return '{} running'.format(coro_name)
+        else:
+            return coro_name
+
     coro_name = None
     if isinstance(coro, CoroWrapper):
         func = coro.func
@@ -271,7 +306,7 @@
         func = coro
 
     if coro_name is None:
-        coro_name = events._format_callback(func, ())
+        coro_name = events._format_callback(func, (), {})
 
     try:
         coro_code = coro.gi_code
diff --git a/lib-python/3/asyncio/events.py b/lib-python/3/asyncio/events.py
--- a/lib-python/3/asyncio/events.py
+++ b/lib-python/3/asyncio/events.py
@@ -6,6 +6,7 @@
            'get_event_loop_policy', 'set_event_loop_policy',
            'get_event_loop', 'set_event_loop', 'new_event_loop',
            'get_child_watcher', 'set_child_watcher',
+           '_set_running_loop', '_get_running_loop',
            ]
 
 import functools
@@ -35,23 +36,25 @@
     return None
 
 
-def _format_args(args):
-    """Format function arguments.
+def _format_args_and_kwargs(args, kwargs):
+    """Format function arguments and keyword arguments.
 
     Special case for a single parameter: ('hello',) is formatted as ('hello').
     """
     # use reprlib to limit the length of the output
-    args_repr = reprlib.repr(args)
-    if len(args) == 1 and args_repr.endswith(',)'):
-        args_repr = args_repr[:-2] + ')'
-    return args_repr
+    items = []
+    if args:
+        items.extend(reprlib.repr(arg) for arg in args)
+    if kwargs:
+        items.extend('{}={}'.format(k, reprlib.repr(v))
+                     for k, v in kwargs.items())
+    return '(' + ', '.join(items) + ')'
 
 
-def _format_callback(func, args, suffix=''):
+def _format_callback(func, args, kwargs, suffix=''):
     if isinstance(func, functools.partial):
-        if args is not None:
-            suffix = _format_args(args) + suffix
-        return _format_callback(func.func, func.args, suffix)
+        suffix = _format_args_and_kwargs(args, kwargs) + suffix
+        return _format_callback(func.func, func.args, func.keywords, suffix)
 
     if hasattr(func, '__qualname__'):
         func_repr = getattr(func, '__qualname__')
@@ -60,14 +63,13 @@
     else:
         func_repr = repr(func)
 
-    if args is not None:
-        func_repr += _format_args(args)
+    func_repr += _format_args_and_kwargs(args, kwargs)
     if suffix:
         func_repr += suffix
     return func_repr
 
 def _format_callback_source(func, args):
-    func_repr = _format_callback(func, args)
+    func_repr = _format_callback(func, args, None)
     source = _get_function_source(func)
     if source:
         func_repr += ' at %s:%s' % source
@@ -81,7 +83,6 @@
                  '_source_traceback', '_repr', '__weakref__')
 
     def __init__(self, callback, args, loop):
-        assert not isinstance(callback, Handle), 'A Handle is not a callback'
         self._loop = loop
         self._callback = callback
         self._args = args
@@ -248,6 +249,10 @@
         """
         raise NotImplementedError
 
+    def shutdown_asyncgens(self):
+        """Shutdown all active asynchronous generators."""
+        raise NotImplementedError
+
     # Methods scheduling callbacks.  All these return Handles.
 
     def _timer_handle_cancelled(self, handle):
@@ -603,6 +608,30 @@
 _lock = threading.Lock()
 
 
+# A TLS for the running event loop, used by _get_running_loop.
+class _RunningLoop(threading.local):
+    _loop = None
+_running_loop = _RunningLoop()
+
+
+def _get_running_loop():
+    """Return the running event loop or None.
+
+    This is a low-level function intended to be used by event loops.
+    This function is thread-specific.
+    """
+    return _running_loop._loop
+
+
+def _set_running_loop(loop):
+    """Set the running event loop.
+
+    This is a low-level function intended to be used by event loops.
+    This function is thread-specific.
+    """
+    _running_loop._loop = loop
+
+
 def _init_event_loop_policy():
     global _event_loop_policy
     with _lock:
@@ -628,7 +657,17 @@
 
 
 def get_event_loop():
-    """Equivalent to calling get_event_loop_policy().get_event_loop()."""
+    """Return an asyncio event loop.
+
+    When called from a coroutine or a callback (e.g. scheduled with call_soon
+    or similar API), this function will always return the running event loop.
+
+    If there is no running event loop set, the function will return
+    the result of `get_event_loop_policy().get_event_loop()` call.
+    """
+    current_loop = _get_running_loop()
+    if current_loop is not None:
+        return current_loop
     return get_event_loop_policy().get_event_loop()
 
 
diff --git a/lib-python/3/asyncio/futures.py b/lib-python/3/asyncio/futures.py
--- a/lib-python/3/asyncio/futures.py
+++ b/lib-python/3/asyncio/futures.py
@@ -2,7 +2,7 @@
 
 __all__ = ['CancelledError', 'TimeoutError',
            'InvalidStateError',
-           'Future', 'wrap_future',
+           'Future', 'wrap_future', 'isfuture',
            ]
 
 import concurrent.futures._base
@@ -110,6 +110,17 @@
             self.loop.call_exception_handler({'message': msg})
 
 
+def isfuture(obj):
+    """Check for a Future.
+
+    This returns True when obj is a Future instance or is advertising
+    itself as duck-type compatible by setting _asyncio_future_blocking.
+    See comment in Future for more details.
+    """
+    return (hasattr(obj.__class__, '_asyncio_future_blocking') and
+            obj._asyncio_future_blocking is not None)
+
+
 class Future:
     """This class is *almost* compatible with concurrent.futures.Future.
 
@@ -134,7 +145,15 @@
     _loop = None
     _source_traceback = None
 
-    _blocking = False  # proper use of future (yield vs yield from)
+    # This field is used for a dual purpose:
+    # - Its presence is a marker to declare that a class implements
+    #   the Future protocol (i.e. is intended to be duck-type compatible).
+    #   The value must also be not-None, to enable a subclass to declare
+    #   that it is not compatible by setting this to None.
+    # - It is set by __iter__() below so that Task._step() can tell
+    #   the difference between `yield from Future()` (correct) vs.
+    #   `yield Future()` (incorrect).
+    _asyncio_future_blocking = False
 
     _log_traceback = False   # Used for Python 3.4 and later
     _tb_logger = None        # Used for Python 3.3 only
@@ -357,7 +376,7 @@
 
     def __iter__(self):
         if not self.done():
-            self._blocking = True
+            self._asyncio_future_blocking = True
             yield self  # This tells Task to wait for completion.
         assert self.done(), "yield from wasn't used with future"
         return self.result()  # May raise too.
@@ -415,15 +434,17 @@
     If destination is cancelled, source gets cancelled too.
     Compatible with both asyncio.Future and concurrent.futures.Future.
     """
-    if not isinstance(source, (Future, concurrent.futures.Future)):
+    if not isfuture(source) and not isinstance(source,
+                                               concurrent.futures.Future):
         raise TypeError('A future is required for source argument')
-    if not isinstance(destination, (Future, concurrent.futures.Future)):
+    if not isfuture(destination) and not isinstance(destination,
+                                                    concurrent.futures.Future):
         raise TypeError('A future is required for destination argument')
-    source_loop = source._loop if isinstance(source, Future) else None
-    dest_loop = destination._loop if isinstance(destination, Future) else None
+    source_loop = source._loop if isfuture(source) else None
+    dest_loop = destination._loop if isfuture(destination) else None
 
     def _set_state(future, other):
-        if isinstance(future, Future):
+        if isfuture(future):
             _copy_future_state(other, future)
         else:
             _set_concurrent_future_state(future, other)
@@ -447,7 +468,7 @@
 
 def wrap_future(future, *, loop=None):
     """Wrap concurrent.futures.Future object."""
-    if isinstance(future, Future):
+    if isfuture(future):
         return future
     assert isinstance(future, concurrent.futures.Future), \
         'concurrent.futures.Future is expected, got {!r}'.format(future)
diff --git a/lib-python/3/asyncio/locks.py b/lib-python/3/asyncio/locks.py
--- a/lib-python/3/asyncio/locks.py
+++ b/lib-python/3/asyncio/locks.py
@@ -166,7 +166,7 @@
         This method blocks until the lock is unlocked, then sets it to
         locked and returns True.
         """
-        if not self._waiters and not self._locked:
+        if not self._locked and all(w.cancelled() for w in self._waiters):
             self._locked = True
             return True
 
diff --git a/lib-python/3/asyncio/proactor_events.py b/lib-python/3/asyncio/proactor_events.py
--- a/lib-python/3/asyncio/proactor_events.py
+++ b/lib-python/3/asyncio/proactor_events.py
@@ -66,6 +66,12 @@
     def _set_extra(self, sock):
         self._extra['pipe'] = sock
 
+    def set_protocol(self, protocol):
+        self._protocol = protocol
+
+    def get_protocol(self):
+        return self._protocol
+
     def is_closing(self):
         return self._closing
 
@@ -488,7 +494,7 @@
         self._csock.send(b'\0')
 
     def _start_serving(self, protocol_factory, sock,
-                       sslcontext=None, server=None):
+                       sslcontext=None, server=None, backlog=100):
 
         def loop(f=None):
             try:
diff --git a/lib-python/3/asyncio/queues.py b/lib-python/3/asyncio/queues.py
--- a/lib-python/3/asyncio/queues.py
+++ b/lib-python/3/asyncio/queues.py
@@ -7,7 +7,6 @@
 
 from . import compat
 from . import events
-from . import futures
 from . import locks
 from .coroutines import coroutine
 
diff --git a/lib-python/3/asyncio/selector_events.py b/lib-python/3/asyncio/selector_events.py
--- a/lib-python/3/asyncio/selector_events.py
+++ b/lib-python/3/asyncio/selector_events.py
@@ -11,6 +11,7 @@
 import functools
 import socket
 import warnings
+import weakref
 try:
     import ssl
 except ImportError:  # pragma: no cover
@@ -39,6 +40,17 @@
         return bool(key.events & event)
 
 
+if hasattr(socket, 'TCP_NODELAY'):
+    def _set_nodelay(sock):
+        if (sock.family in {socket.AF_INET, socket.AF_INET6} and
+                sock.type == socket.SOCK_STREAM and
+                sock.proto == socket.IPPROTO_TCP):
+            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+else:
+    def _set_nodelay(sock):
+        pass
+
+
 class BaseSelectorEventLoop(base_events.BaseEventLoop):
     """Selector event loop.
 
@@ -53,6 +65,7 @@
         logger.debug('Using selector: %s', selector.__class__.__name__)
         self._selector = selector
         self._make_self_pipe()
+        self._transports = weakref.WeakValueDictionary()
 
     def _make_socket_transport(self, sock, protocol, waiter=None, *,
                                extra=None, server=None):
@@ -104,7 +117,7 @@
         raise NotImplementedError
 
     def _close_self_pipe(self):
-        self.remove_reader(self._ssock.fileno())
+        self._remove_reader(self._ssock.fileno())
         self._ssock.close()
         self._ssock = None
         self._csock.close()
@@ -117,7 +130,7 @@
         self._ssock.setblocking(False)
         self._csock.setblocking(False)
         self._internal_fds += 1
-        self.add_reader(self._ssock.fileno(), self._read_from_self)
+        self._add_reader(self._ssock.fileno(), self._read_from_self)
 
     def _process_self_data(self, data):
         pass
@@ -151,43 +164,50 @@
                                  exc_info=True)
 
     def _start_serving(self, protocol_factory, sock,
-                       sslcontext=None, server=None):
-        self.add_reader(sock.fileno(), self._accept_connection,
-                        protocol_factory, sock, sslcontext, server)
+                       sslcontext=None, server=None, backlog=100):
+        self._add_reader(sock.fileno(), self._accept_connection,
+                         protocol_factory, sock, sslcontext, server, backlog)
 
     def _accept_connection(self, protocol_factory, sock,
-                           sslcontext=None, server=None):
-        try:
-            conn, addr = sock.accept()
-            if self._debug:
-                logger.debug("%r got a new connection from %r: %r",
-                             server, addr, conn)
-            conn.setblocking(False)
-        except (BlockingIOError, InterruptedError, ConnectionAbortedError):
-            pass  # False alarm.
-        except OSError as exc:
-            # There's nowhere to send the error, so just log it.
-            if exc.errno in (errno.EMFILE, errno.ENFILE,
-                             errno.ENOBUFS, errno.ENOMEM):
-                # Some platforms (e.g. Linux keep reporting the FD as
-                # ready, so we remove the read handler temporarily.
-                # We'll try again in a while.
-                self.call_exception_handler({
-                    'message': 'socket.accept() out of system resource',
-                    'exception': exc,
-                    'socket': sock,
-                })
-                self.remove_reader(sock.fileno())
-                self.call_later(constants.ACCEPT_RETRY_DELAY,
-                                self._start_serving,
-                                protocol_factory, sock, sslcontext, server)
+                           sslcontext=None, server=None, backlog=100):
+        # This method is only called once for each event loop tick where the
+        # listening socket has triggered an EVENT_READ. There may be multiple
+        # connections waiting for an .accept() so it is called in a loop.
+        # See https://bugs.python.org/issue27906 for more details.
+        for _ in range(backlog):
+            try:
+                conn, addr = sock.accept()
+                if self._debug:
+                    logger.debug("%r got a new connection from %r: %r",
+                                 server, addr, conn)
+                conn.setblocking(False)
+            except (BlockingIOError, InterruptedError, ConnectionAbortedError):
+                # Early exit because the socket accept buffer is empty.
+                return None
+            except OSError as exc:
+                # There's nowhere to send the error, so just log it.
+                if exc.errno in (errno.EMFILE, errno.ENFILE,
+                                 errno.ENOBUFS, errno.ENOMEM):
+                    # Some platforms (e.g. Linux keep reporting the FD as
+                    # ready, so we remove the read handler temporarily.
+                    # We'll try again in a while.
+                    self.call_exception_handler({
+                        'message': 'socket.accept() out of system resource',
+                        'exception': exc,
+                        'socket': sock,
+                    })
+                    self._remove_reader(sock.fileno())
+                    self.call_later(constants.ACCEPT_RETRY_DELAY,
+                                    self._start_serving,
+                                    protocol_factory, sock, sslcontext, server,
+                                    backlog)
+                else:
+                    raise  # The event loop will catch, log and ignore it.
             else:
-                raise  # The event loop will catch, log and ignore it.
-        else:
-            extra = {'peername': addr}
-            accept = self._accept_connection2(protocol_factory, conn, extra,
-                                              sslcontext, server)
-            self.create_task(accept)
+                extra = {'peername': addr}
+                accept = self._accept_connection2(protocol_factory, conn, extra,
+                                                  sslcontext, server)
+                self.create_task(accept)
 
     @coroutine
     def _accept_connection2(self, protocol_factory, conn, extra,
@@ -226,8 +246,18 @@
                     context['transport'] = transport
                 self.call_exception_handler(context)
 
-    def add_reader(self, fd, callback, *args):
-        """Add a reader callback."""
+    def _ensure_fd_no_transport(self, fd):
+        try:
+            transport = self._transports[fd]
+        except KeyError:
+            pass
+        else:
+            if not transport.is_closing():
+                raise RuntimeError(
+                    'File descriptor {!r} is used by transport {!r}'.format(
+                        fd, transport))
+
+    def _add_reader(self, fd, callback, *args):
         self._check_closed()
         handle = events.Handle(callback, args, self)
         try:
@@ -242,8 +272,7 @@
             if reader is not None:
                 reader.cancel()
 
-    def remove_reader(self, fd):
-        """Remove a reader callback."""
+    def _remove_reader(self, fd):
         if self.is_closed():
             return False
         try:
@@ -264,8 +293,7 @@
             else:
                 return False
 
-    def add_writer(self, fd, callback, *args):
-        """Add a writer callback.."""
+    def _add_writer(self, fd, callback, *args):
         self._check_closed()
         handle = events.Handle(callback, args, self)
         try:
@@ -280,7 +308,7 @@
             if writer is not None:
                 writer.cancel()
 
-    def remove_writer(self, fd):
+    def _remove_writer(self, fd):
         """Remove a writer callback."""
         if self.is_closed():
             return False
@@ -303,6 +331,26 @@
             else:
                 return False
 
+    def add_reader(self, fd, callback, *args):
+        """Add a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_reader(fd, callback, *args)
+
+    def remove_reader(self, fd):
+        """Remove a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_reader(fd)
+
+    def add_writer(self, fd, callback, *args):
+        """Add a writer callback.."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_writer(fd, callback, *args)
+
+    def remove_writer(self, fd):
+        """Remove a writer callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_writer(fd)
+
     def sock_recv(self, sock, n):
         """Receive data from the socket.
 
@@ -382,6 +430,7 @@
                 data = data[n:]
             self.add_writer(fd, self._sock_sendall, fut, True, sock, data)
 
+    @coroutine
     def sock_connect(self, sock, address):
         """Connect to a remote socket at address.
 
@@ -390,23 +439,16 @@
         if self._debug and sock.gettimeout() != 0:
             raise ValueError("the socket must be non-blocking")
 
+        if not hasattr(socket, 'AF_UNIX') or sock.family != socket.AF_UNIX:
+            resolved = base_events._ensure_resolved(
+                address, family=sock.family, proto=sock.proto, loop=self)
+            if not resolved.done():
+                yield from resolved
+            _, _, _, _, address = resolved.result()[0]
+
         fut = self.create_future()
-        if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX:
-            self._sock_connect(fut, sock, address)
-        else:
-            resolved = base_events._ensure_resolved(address, loop=self)
-            resolved.add_done_callback(
-                lambda resolved: self._on_resolved(fut, sock, resolved))
-
-        return fut
-
-    def _on_resolved(self, fut, sock, resolved):
-        try:
-            _, _, _, _, address = resolved.result()[0]
-        except Exception as exc:
-            fut.set_exception(exc)
-        else:
-            self._sock_connect(fut, sock, address)
+        self._sock_connect(fut, sock, address)
+        return (yield from fut)
 
     def _sock_connect(self, fut, sock, address):
         fd = sock.fileno()
@@ -417,8 +459,8 @@
             # connection runs in background. We have to wait until the socket
             # becomes writable to be notified when the connection succeed or
             # fails.
-            fut.add_done_callback(functools.partial(self._sock_connect_done,
-                                                    fd))
+            fut.add_done_callback(
+                functools.partial(self._sock_connect_done, fd))
             self.add_writer(fd, self._sock_connect_cb, fut, sock, address)
         except Exception as exc:
             fut.set_exception(exc)
@@ -482,17 +524,17 @@
             fileobj, (reader, writer) = key.fileobj, key.data
             if mask & selectors.EVENT_READ and reader is not None:
                 if reader._cancelled:
-                    self.remove_reader(fileobj)
+                    self._remove_reader(fileobj)
                 else:
                     self._add_callback(reader)
             if mask & selectors.EVENT_WRITE and writer is not None:
                 if writer._cancelled:
-                    self.remove_writer(fileobj)
+                    self._remove_writer(fileobj)
                 else:
                     self._add_callback(writer)
 
     def _stop_serving(self, sock):
-        self.remove_reader(sock.fileno())
+        self._remove_reader(sock.fileno())
         sock.close()
 
 
@@ -527,6 +569,7 @@
         self._closing = False  # Set when close() called.
         if self._server is not None:
             self._server._attach()
+        loop._transports[self._sock_fd] = self
 
     def __repr__(self):
         info = [self.__class__.__name__]
@@ -559,6 +602,12 @@
     def abort(self):
         self._force_close(None)
 
+    def set_protocol(self, protocol):
+        self._protocol = protocol
+
+    def get_protocol(self):
+        return self._protocol
+
     def is_closing(self):
         return self._closing
 
@@ -566,10 +615,10 @@
         if self._closing:
             return
         self._closing = True
-        self._loop.remove_reader(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
         if not self._buffer:
             self._conn_lost += 1
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             self._loop.call_soon(self._call_connection_lost, None)
 
     # On Python 3.3 and older, objects with a destructor part of a reference
@@ -600,10 +649,10 @@
             return
         if self._buffer:
             self._buffer.clear()
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
         if not self._closing:
             self._closing = True
-            self._loop.remove_reader(self._sock_fd)
+            self._loop._remove_reader(self._sock_fd)
         self._conn_lost += 1
         self._loop.call_soon(self._call_connection_lost, exc)
 
@@ -633,9 +682,14 @@
         self._eof = False
         self._paused = False
 
+        # Disable the Nagle algorithm -- small writes will be
+        # sent without waiting for the TCP ACK.  This generally
+        # decreases the latency (in some cases significantly.)
+        _set_nodelay(self._sock)
+
         self._loop.call_soon(self._protocol.connection_made, self)
         # only start reading when connection_made() has been called
-        self._loop.call_soon(self._loop.add_reader,
+        self._loop.call_soon(self._loop._add_reader,
                              self._sock_fd, self._read_ready)
         if waiter is not None:
             # only wake up the waiter when connection_made() has been called
@@ -648,7 +702,7 @@
         if self._paused:
             raise RuntimeError('Already paused')
         self._paused = True
-        self._loop.remove_reader(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
         if self._loop.get_debug():
             logger.debug("%r pauses reading", self)
 
@@ -658,7 +712,7 @@
         self._paused = False
         if self._closing:
             return
-        self._loop.add_reader(self._sock_fd, self._read_ready)
+        self._loop._add_reader(self._sock_fd, self._read_ready)
         if self._loop.get_debug():
             logger.debug("%r resumes reading", self)
 
@@ -682,7 +736,7 @@
                     # We're keeping the connection open so the
                     # protocol can write more, but we still can't
                     # receive more, so remove the reader callback.
-                    self._loop.remove_reader(self._sock_fd)
+                    self._loop._remove_reader(self._sock_fd)
                 else:
                     self.close()
 
@@ -715,7 +769,7 @@
                 if not data:
                     return
             # Not all was written; register write handler.
-            self._loop.add_writer(self._sock_fd, self._write_ready)
+            self._loop._add_writer(self._sock_fd, self._write_ready)
 
         # Add it to the buffer.
         self._buffer.extend(data)
@@ -731,7 +785,7 @@
         except (BlockingIOError, InterruptedError):
             pass
         except Exception as exc:
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             self._buffer.clear()
             self._fatal_error(exc, 'Fatal write error on socket transport')
         else:
@@ -739,7 +793,7 @@
                 del self._buffer[:n]
             self._maybe_resume_protocol()  # May append to buffer.
             if not self._buffer:
-                self._loop.remove_writer(self._sock_fd)
+                self._loop._remove_writer(self._sock_fd)
                 if self._closing:
                     self._call_connection_lost(None)
                 elif self._eof:
@@ -810,19 +864,19 @@
         try:
             self._sock.do_handshake()
         except ssl.SSLWantReadError:
-            self._loop.add_reader(self._sock_fd,
-                                  self._on_handshake, start_time)
+            self._loop._add_reader(self._sock_fd,
+                                   self._on_handshake, start_time)
             return
         except ssl.SSLWantWriteError:
-            self._loop.add_writer(self._sock_fd,
-                                  self._on_handshake, start_time)
+            self._loop._add_writer(self._sock_fd,
+                                   self._on_handshake, start_time)
             return
         except BaseException as exc:
             if self._loop.get_debug():
                 logger.warning("%r: SSL handshake failed",
                                self, exc_info=True)
-            self._loop.remove_reader(self._sock_fd)
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_reader(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             self._sock.close()
             self._wakeup_waiter(exc)
             if isinstance(exc, Exception):
@@ -830,8 +884,8 @@
             else:
                 raise
 
-        self._loop.remove_reader(self._sock_fd)
-        self._loop.remove_writer(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
+        self._loop._remove_writer(self._sock_fd)
 
         peercert = self._sock.getpeercert()
         if not hasattr(self._sslcontext, 'check_hostname'):
@@ -859,7 +913,7 @@
 
         self._read_wants_write = False
         self._write_wants_read = False
-        self._loop.add_reader(self._sock_fd, self._read_ready)
+        self._loop._add_reader(self._sock_fd, self._read_ready)
         self._protocol_connected = True
         self._loop.call_soon(self._protocol.connection_made, self)
         # only wake up the waiter when connection_made() has been called
@@ -881,7 +935,7 @@
         if self._paused:
             raise RuntimeError('Already paused')
         self._paused = True
-        self._loop.remove_reader(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
         if self._loop.get_debug():
             logger.debug("%r pauses reading", self)
 
@@ -891,7 +945,7 @@
         self._paused = False
         if self._closing:
             return
-        self._loop.add_reader(self._sock_fd, self._read_ready)
+        self._loop._add_reader(self._sock_fd, self._read_ready)
         if self._loop.get_debug():
             logger.debug("%r resumes reading", self)
 
@@ -903,7 +957,7 @@
             self._write_ready()
 
             if self._buffer:
-                self._loop.add_writer(self._sock_fd, self._write_ready)
+                self._loop._add_writer(self._sock_fd, self._write_ready)
 
         try:
             data = self._sock.recv(self.max_size)
@@ -911,8 +965,8 @@
             pass
         except ssl.SSLWantWriteError:
             self._read_wants_write = True
-            self._loop.remove_reader(self._sock_fd)
-            self._loop.add_writer(self._sock_fd, self._write_ready)
+            self._loop._remove_reader(self._sock_fd)
+            self._loop._add_writer(self._sock_fd, self._write_ready)
         except Exception as exc:
             self._fatal_error(exc, 'Fatal read error on SSL transport')
         else:
@@ -937,7 +991,7 @@
             self._read_ready()
 
             if not (self._paused or self._closing):
-                self._loop.add_reader(self._sock_fd, self._read_ready)
+                self._loop._add_reader(self._sock_fd, self._read_ready)
 
         if self._buffer:
             try:
@@ -946,10 +1000,10 @@
                 n = 0
             except ssl.SSLWantReadError:
                 n = 0
-                self._loop.remove_writer(self._sock_fd)
+                self._loop._remove_writer(self._sock_fd)
                 self._write_wants_read = True
             except Exception as exc:
-                self._loop.remove_writer(self._sock_fd)
+                self._loop._remove_writer(self._sock_fd)
                 self._buffer.clear()
                 self._fatal_error(exc, 'Fatal write error on SSL transport')
                 return
@@ -960,7 +1014,7 @@
         self._maybe_resume_protocol()  # May append to buffer.
 
         if not self._buffer:
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             if self._closing:
                 self._call_connection_lost(None)
 
@@ -978,7 +1032,7 @@
             return
 
         if not self._buffer:
-            self._loop.add_writer(self._sock_fd, self._write_ready)
+            self._loop._add_writer(self._sock_fd, self._write_ready)
 
         # Add it to the buffer.
         self._buffer.extend(data)
@@ -998,7 +1052,7 @@
         self._address = address
         self._loop.call_soon(self._protocol.connection_made, self)
         # only start reading when connection_made() has been called
-        self._loop.call_soon(self._loop.add_reader,
+        self._loop.call_soon(self._loop._add_reader,
                              self._sock_fd, self._read_ready)
         if waiter is not None:
             # only wake up the waiter when connection_made() has been called
@@ -1048,7 +1102,7 @@
                     self._sock.sendto(data, addr)
                 return
             except (BlockingIOError, InterruptedError):
-                self._loop.add_writer(self._sock_fd, self._sendto_ready)
+                self._loop._add_writer(self._sock_fd, self._sendto_ready)
             except OSError as exc:
                 self._protocol.error_received(exc)
                 return
@@ -1082,6 +1136,6 @@
 
         self._maybe_resume_protocol()  # May append to buffer.
         if not self._buffer:
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             if self._closing:
                 self._call_connection_lost(None)
diff --git a/lib-python/3/asyncio/sslproto.py b/lib-python/3/asyncio/sslproto.py
--- a/lib-python/3/asyncio/sslproto.py
+++ b/lib-python/3/asyncio/sslproto.py
@@ -5,6 +5,7 @@
 except ImportError:  # pragma: no cover
     ssl = None
 
+from . import base_events
 from . import compat
 from . import protocols
 from . import transports
@@ -304,6 +305,12 @@
         """Get optional transport information."""
         return self._ssl_protocol._get_extra_info(name, default)
 
+    def set_protocol(self, protocol):
+        self._app_protocol = protocol
+
+    def get_protocol(self):
+        return self._app_protocol
+
     def is_closing(self):
         return self._closed
 
@@ -403,7 +410,8 @@
     """
 
     def __init__(self, loop, app_protocol, sslcontext, waiter,
-                 server_side=False, server_hostname=None):
+                 server_side=False, server_hostname=None,
+                 call_connection_made=True):
         if ssl is None:
             raise RuntimeError('stdlib ssl module not available')
 
@@ -436,6 +444,7 @@
         self._in_shutdown = False
         # transport, ex: SelectorSocketTransport
         self._transport = None
+        self._call_connection_made = call_connection_made
 
     def _wakeup_waiter(self, exc=None):
         if self._waiter is None:
@@ -470,6 +479,7 @@
             self._loop.call_soon(self._app_protocol.connection_lost, exc)
         self._transport = None
         self._app_transport = None
+        self._wakeup_waiter(exc)
 
     def pause_writing(self):
         """Called when the low-level transport's buffer goes over
@@ -599,7 +609,8 @@
                            compression=sslobj.compression(),
                            ssl_object=sslobj,
                            )
-        self._app_protocol.connection_made(self._app_transport)
+        if self._call_connection_made:
+            self._app_protocol.connection_made(self._app_transport)
         self._wakeup_waiter()
         self._session_established = True
         # In case transport.write() was already called. Don't call
diff --git a/lib-python/3/asyncio/streams.py b/lib-python/3/asyncio/streams.py
--- a/lib-python/3/asyncio/streams.py
+++ b/lib-python/3/asyncio/streams.py
@@ -448,6 +448,7 @@
         assert not self._eof, '_wait_for_data after EOF'
 
         # Waiting for data while paused will make deadlock, so prevent it.
+        # This is essential for readexactly(n) for case when n > self._limit.
         if self._paused:
             self._paused = False
             self._transport.resume_reading()
@@ -590,7 +591,7 @@
         bytes. If the EOF was received and the internal buffer is empty, return
         an empty bytes object.
 
-        If n is zero, return empty bytes object immediatelly.
+        If n is zero, return empty bytes object immediately.
 
         If n is positive, this function try to read `n` bytes, and may return
         less or equal bytes than requested, but at least one byte. If EOF was
@@ -658,25 +659,22 @@
         if n == 0:
             return b''
 
-        # There used to be "optimized" code here.  It created its own
-        # Future and waited until self._buffer had at least the n
-        # bytes, then called read(n).  Unfortunately, this could pause
-        # the transport if the argument was larger than the pause
-        # limit (which is twice self._limit).  So now we just read()
-        # into a local buffer.
+        while len(self._buffer) < n:
+            if self._eof:
+                incomplete = bytes(self._buffer)
+                self._buffer.clear()
+                raise IncompleteReadError(incomplete, n)
 
-        blocks = []
-        while n > 0:
-            block = yield from self.read(n)
-            if not block:
-                partial = b''.join(blocks)
-                raise IncompleteReadError(partial, len(partial) + n)
-            blocks.append(block)
-            n -= len(block)
+            yield from self._wait_for_data('readexactly')
 
-        assert n == 0
-
-        return b''.join(blocks)
+        if len(self._buffer) == n:
+            data = bytes(self._buffer)
+            self._buffer.clear()
+        else:
+            data = bytes(self._buffer[:n])
+            del self._buffer[:n]
+        self._maybe_resume_transport()
+        return data
 
     if compat.PY35:
         @coroutine
diff --git a/lib-python/3/asyncio/tasks.py b/lib-python/3/asyncio/tasks.py
--- a/lib-python/3/asyncio/tasks.py
+++ b/lib-python/3/asyncio/tasks.py
@@ -241,7 +241,7 @@
                 result = coro.throw(exc)
         except StopIteration as exc:
             self.set_result(exc.value)
-        except futures.CancelledError as exc:
+        except futures.CancelledError:
             super().cancel()  # I.e., Future.cancel(self).
         except Exception as exc:
             self.set_exception(exc)
@@ -249,7 +249,8 @@
             self.set_exception(exc)
             raise
         else:
-            if isinstance(result, futures.Future):
+            blocking = getattr(result, '_asyncio_future_blocking', None)
+            if blocking is not None:
                 # Yielded Future must come from Future.__iter__().
                 if result._loop is not self._loop:
                     self._loop.call_soon(
@@ -257,13 +258,20 @@
                         RuntimeError(
                             'Task {!r} got Future {!r} attached to a '
                             'different loop'.format(self, result)))
-                elif result._blocking:
-                    result._blocking = False
-                    result.add_done_callback(self._wakeup)
-                    self._fut_waiter = result
-                    if self._must_cancel:
-                        if self._fut_waiter.cancel():
-                            self._must_cancel = False
+                elif blocking:
+                    if result is self:
+                        self._loop.call_soon(
+                            self._step,
+                            RuntimeError(
+                                'Task cannot await on itself: {!r}'.format(
+                                    self)))
+                    else:
+                        result._asyncio_future_blocking = False
+                        result.add_done_callback(self._wakeup)
+                        self._fut_waiter = result
+                        if self._must_cancel:
+                            if self._fut_waiter.cancel():
+                                self._must_cancel = False
                 else:
                     self._loop.call_soon(
                         self._step,
@@ -332,7 +340,7 @@
     Note: This does not raise TimeoutError! Futures that aren't done
     when the timeout occurs are returned in the second set.
     """
-    if isinstance(fs, futures.Future) or coroutines.iscoroutine(fs):
+    if futures.isfuture(fs) or coroutines.iscoroutine(fs):
         raise TypeError("expect a list of futures, not %s" % type(fs).__name__)
     if not fs:
         raise ValueError('Set of coroutines/Futures is empty.')
@@ -461,7 +469,7 @@
 
     Note: The futures 'f' are not necessarily members of fs.
     """
-    if isinstance(fs, futures.Future) or coroutines.iscoroutine(fs):
+    if futures.isfuture(fs) or coroutines.iscoroutine(fs):
         raise TypeError("expect a list of futures, not %s" % type(fs).__name__)
     loop = loop if loop is not None else events.get_event_loop()
     todo = {ensure_future(f, loop=loop) for f in set(fs)}
@@ -518,7 +526,7 @@
         h.cancel()
 
 
-def async(coro_or_future, *, loop=None):
+def async_(coro_or_future, *, loop=None):
     """Wrap a coroutine in a future.
 
     If the argument is a Future, it is returned directly.
@@ -531,13 +539,18 @@
 
     return ensure_future(coro_or_future, loop=loop)
 
+# Silence DeprecationWarning:
+globals()['async'] = async_
+async_.__name__ = 'async'
+del async_
+
 
 def ensure_future(coro_or_future, *, loop=None):
     """Wrap a coroutine or an awaitable in a future.
 
     If the argument is a Future, it is returned directly.
     """
-    if isinstance(coro_or_future, futures.Future):
+    if futures.isfuture(coro_or_future):
         if loop is not None and loop is not coro_or_future._loop:
             raise ValueError('loop argument must agree with Future')
         return coro_or_future
@@ -579,15 +592,21 @@
     def cancel(self):
         if self.done():
             return False
+        ret = False
         for child in self._children:
-            child.cancel()
-        return True
+            if child.cancel():
+                ret = True
+        return ret
 
 
 def gather(*coros_or_futures, loop=None, return_exceptions=False):
     """Return a future aggregating results from the given coroutines
     or futures.
 
+    Coroutines will be wrapped in a future and scheduled in the event
+    loop. They will not necessarily be scheduled in the same order as
+    passed in.
+
     All futures must share the same event loop.  If all the tasks are
     done successfully, the returned future's result is the list of
     results (in the order of the original sequence, not necessarily
@@ -613,7 +632,7 @@
 
     arg_to_fut = {}
     for arg in set(coros_or_futures):
-        if not isinstance(arg, futures.Future):
+        if not futures.isfuture(arg):
             fut = ensure_future(arg, loop=loop)
             if loop is None:
                 loop = fut._loop
diff --git a/lib-python/3/asyncio/test_utils.py b/lib-python/3/asyncio/test_utils.py
--- a/lib-python/3/asyncio/test_utils.py
+++ b/lib-python/3/asyncio/test_utils.py
@@ -13,6 +13,8 @@
 import threading
 import time
 import unittest
+import weakref
+
 from unittest import mock
 
 from http.server import HTTPServer
@@ -300,6 +302,8 @@
         self.writers = {}
         self.reset_counters()
 
+        self._transports = weakref.WeakValueDictionary()
+
     def time(self):
         return self._time
 
@@ -318,10 +322,10 @@
             else:  # pragma: no cover
                 raise AssertionError("Time generator is not finished")
 
-    def add_reader(self, fd, callback, *args):
+    def _add_reader(self, fd, callback, *args):
         self.readers[fd] = events.Handle(callback, args, self)
 
-    def remove_reader(self, fd):
+    def _remove_reader(self, fd):
         self.remove_reader_count[fd] += 1
         if fd in self.readers:
             del self.readers[fd]
@@ -337,10 +341,10 @@
         assert handle._args == args, '{!r} != {!r}'.format(
             handle._args, args)
 
-    def add_writer(self, fd, callback, *args):
+    def _add_writer(self, fd, callback, *args):
         self.writers[fd] = events.Handle(callback, args, self)
 
-    def remove_writer(self, fd):
+    def _remove_writer(self, fd):
         self.remove_writer_count[fd] += 1
         if fd in self.writers:
             del self.writers[fd]
@@ -356,6 +360,36 @@
         assert handle._args == args, '{!r} != {!r}'.format(
             handle._args, args)
 
+    def _ensure_fd_no_transport(self, fd):
+        try:
+            transport = self._transports[fd]
+        except KeyError:
+            pass
+        else:
+            raise RuntimeError(
+                'File descriptor {!r} is used by transport {!r}'.format(
+                    fd, transport))
+
+    def add_reader(self, fd, callback, *args):
+        """Add a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_reader(fd, callback, *args)
+
+    def remove_reader(self, fd):
+        """Remove a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_reader(fd)
+
+    def add_writer(self, fd, callback, *args):
+        """Add a writer callback.."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_writer(fd, callback, *args)
+
+    def remove_writer(self, fd):
+        """Remove a writer callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_writer(fd)
+
     def reset_counters(self):
         self.remove_reader_count = collections.defaultdict(int)
         self.remove_writer_count = collections.defaultdict(int)
@@ -415,7 +449,13 @@
         self.set_event_loop(loop)
         return loop
 
+    def setUp(self):
+        self._get_running_loop = events._get_running_loop
+        events._get_running_loop = lambda: None
+
     def tearDown(self):
+        events._get_running_loop = self._get_running_loop
+
         events.set_event_loop(None)
 
         # Detect CPython bug #23353: ensure that yield/yield-from is not used
diff --git a/lib-python/3/asyncio/transports.py b/lib-python/3/asyncio/transports.py
--- a/lib-python/3/asyncio/transports.py
+++ b/lib-python/3/asyncio/transports.py
@@ -33,6 +33,14 @@
         """
         raise NotImplementedError
 
+    def set_protocol(self, protocol):
+        """Set a new protocol."""
+        raise NotImplementedError
+
+    def get_protocol(self):
+        """Return the current protocol."""
+        raise NotImplementedError
+
 
 class ReadTransport(BaseTransport):
     """Interface for read-only transports."""
diff --git a/lib-python/3/asyncio/unix_events.py b/lib-python/3/asyncio/unix_events.py
--- a/lib-python/3/asyncio/unix_events.py
+++ b/lib-python/3/asyncio/unix_events.py
@@ -39,6 +39,13 @@
     pass
 
 
+try:
+    _fspath = os.fspath
+except AttributeError:
+    # Python 3.5 or earlier
+    _fspath = lambda path: path
+
+
 class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop):
     """Unix event loop.
 
@@ -234,6 +241,11 @@
         else:
             if sock is None:
                 raise ValueError('no path and sock were specified')
+            if (sock.family != socket.AF_UNIX or
+                    not base_events._is_stream_socket(sock)):
+                raise ValueError(
+                    'A UNIX Domain Stream Socket was expected, got {!r}'
+                    .format(sock))
             sock.setblocking(False)
 
         transport, protocol = yield from self._create_connection_transport(
@@ -251,8 +263,20 @@
                 raise ValueError(
                     'path and sock can not be specified at the same time')
 
+            path = _fspath(path)
             sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 
+            # Check for abstract socket. `str` and `bytes` paths are supported.
+            if path[0] not in (0, '\x00'):
+                try:
+                    if stat.S_ISSOCK(os.stat(path).st_mode):
+                        os.remove(path)
+                except FileNotFoundError:
+                    pass
+                except OSError as err:
+                    # Directory may have permissions only to create socket.
+                    logger.error('Unable to check or remove stale UNIX socket %r: %r', path, err)
+
             try:
                 sock.bind(path)
             except OSError as exc:
@@ -272,9 +296,11 @@
                 raise ValueError(
                     'path was not specified, and no sock specified')
 
-            if sock.family != socket.AF_UNIX:
+            if (sock.family != socket.AF_UNIX or
+                    not base_events._is_stream_socket(sock)):
                 raise ValueError(
-                    'A UNIX Domain Socket was expected, got {!r}'.format(sock))
+                    'A UNIX Domain Stream Socket was expected, got {!r}'
+                    .format(sock))
 
         server = base_events.Server(self, [sock])
         sock.listen(backlog)
@@ -305,17 +331,23 @@
         self._loop = loop
         self._pipe = pipe
         self._fileno = pipe.fileno()
+        self._protocol = protocol
+        self._closing = False
+
         mode = os.fstat(self._fileno).st_mode
         if not (stat.S_ISFIFO(mode) or
                 stat.S_ISSOCK(mode) or
                 stat.S_ISCHR(mode)):
+            self._pipe = None
+            self._fileno = None
+            self._protocol = None
             raise ValueError("Pipe transport is for pipes/sockets only.")
+
         _set_nonblocking(self._fileno)
-        self._protocol = protocol
-        self._closing = False
+
         self._loop.call_soon(self._protocol.connection_made, self)
         # only start reading when connection_made() has been called
-        self._loop.call_soon(self._loop.add_reader,
+        self._loop.call_soon(self._loop._add_reader,
                              self._fileno, self._read_ready)
         if waiter is not None:
             # only wake up the waiter when connection_made() has been called

From pypy.commits at gmail.com  Sun Feb  5 14:44:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 05 Feb 2017 11:44:58 -0800 (PST)
Subject: [pypy-commit] pypy default: Move buffer functions to
 pypy.module.cpyext.buffer
Message-ID: <589780ba.d185df0a.ad9ab.7662@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r89953:ad2c1430f189
Date: 2017-02-05 19:38 +0000
http://bitbucket.org/pypy/pypy/changeset/ad2c1430f189/

Log:	Move buffer functions to pypy.module.cpyext.buffer

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -126,7 +126,7 @@
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
-PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,7 +1,10 @@
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts)
-from pypy.module.cpyext.pyobject import PyObject
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts, Py_buffer,
+    Py_ssize_t, Py_ssize_tP,
+    PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES)
+from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -11,7 +14,52 @@
     if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
         return 1
     name = rffi.charp2str(cts.cast('char*', pyobj.c_ob_type.c_tp_name))
-    if  name in ('str', 'bytes'):
+    if name in ('str', 'bytes'):
         # XXX remove once wrapper of __buffer__ -> bf_getbuffer works
         return 1
     return 0
+
+ at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
+              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
+def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
+    """
+    Fills in a buffer-info structure correctly for an exporter that can only
+    share a contiguous chunk of memory of "unsigned bytes" of the given
+    length. Returns 0 on success and -1 (with raising an error) on error.
+    """
+    if flags & PyBUF_WRITABLE and readonly:
+        raise oefmt(space.w_ValueError, "Object is not writable")
+    view.c_buf = buf
+    view.c_len = length
+    view.c_obj = obj
+    if obj:
+        Py_IncRef(space, obj)
+    view.c_itemsize = 1
+    rffi.setintfield(view, 'c_readonly', readonly)
+    rffi.setintfield(view, 'c_ndim', 1)
+    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
+    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
+    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
+    return 0
+
+
+ at cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
+def PyBuffer_Release(space, view):
+    """
+    Release the buffer view. This should be called when the buffer is
+    no longer being used as it may free memory from it
+    """
+    Py_DecRef(space, view.c_obj)
+    view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,13 +1,12 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function,
-    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
+    PyVarObject, size_t, slot_function,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
-    PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
-    get_typedescr, _Py_NewReference)
+    PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef,
+    get_typedescr)
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
@@ -476,51 +475,3 @@
     with rffi.scoped_nonmovingbuffer(data) as buf:
         fwrite(buf, 1, count, fp)
     return 0
-
-
-PyBUF_WRITABLE = 0x0001  # Copied from object.h
-
- at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
-              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
-def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
-    """
-    Fills in a buffer-info structure correctly for an exporter that can only
-    share a contiguous chunk of memory of "unsigned bytes" of the given
-    length. Returns 0 on success and -1 (with raising an error) on error.
-    """
-    if flags & PyBUF_WRITABLE and readonly:
-        raise oefmt(space.w_ValueError, "Object is not writable")
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    if obj:
-        Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 1)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
-        view.c_format = rffi.str2charp("B")
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_ND) == PyBUF_ND:
-        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-        view.c_shape[0] = view.c_len
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
-        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-        view.c_strides[0] = view.c_itemsize
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
-    return 0
-
-
- at cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
-def PyBuffer_Release(space, view):
-    """
-    Release the buffer view. This should be called when the buffer is
-    no longer being used as it may free memory from it
-    """
-    Py_DecRef(space, view.c_obj)
-    view.c_obj = lltype.nullptr(PyObject.TO)
-    # XXX do other fields leak memory?

From pypy.commits at gmail.com  Sun Feb  5 15:15:41 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 05 Feb 2017 12:15:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Move buffer functions to
 pypy.module.cpyext.buffer
Message-ID: <589787ed.63afdf0a.703de.a6ae@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89954:1a3f307b706d
Date: 2017-02-05 20:15 +0000
http://bitbucket.org/pypy/pypy/changeset/1a3f307b706d/

Log:	Move buffer functions to pypy.module.cpyext.buffer

diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -40,6 +40,7 @@
 import pypy.module.cpyext.pyerrors
 import pypy.module.cpyext.typeobject
 import pypy.module.cpyext.object
+import pypy.module.cpyext.buffer
 import pypy.module.cpyext.bytesobject
 import pypy.module.cpyext.bytearrayobject
 import pypy.module.cpyext.tupleobject
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -126,7 +126,7 @@
 Py_TPFLAGS_HEAPTYPE
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
-PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/buffer.py
@@ -0,0 +1,72 @@
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.interpreter.error import oefmt
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP,
+    generic_cpy_call,
+    PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES, PyBUF_SIMPLE)
+from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
+
+ at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
+def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
+    """Returns a pointer to a read-only memory location usable as
+    character-based input.  The obj argument must support the single-segment
+    character buffer interface.  On success, returns 0, sets buffer to the
+    memory location and size to the buffer length.  Returns -1 and sets a
+    TypeError on error.
+    """
+    pto = obj.c_ob_type
+    pb = pto.c_tp_as_buffer
+    if not (pb and pb.c_bf_getbuffer):
+        raise oefmt(space.w_TypeError,
+                    "expected an object with the buffer interface")
+    with lltype.scoped_alloc(Py_buffer) as view:
+        ret = generic_cpy_call(
+            space, pb.c_bf_getbuffer,
+            obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
+        if rffi.cast(lltype.Signed, ret) == -1:
+            return -1
+
+        bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
+        sizep[0] = view.c_len
+
+        if pb.c_bf_releasebuffer:
+            generic_cpy_call(space, pb.c_bf_releasebuffer,
+                             obj, view)
+        Py_DecRef(space, view.c_obj)
+    return 0
+
+
+ at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
+              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
+def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
+    """
+    Fills in a buffer-info structure correctly for an exporter that can only
+    share a contiguous chunk of memory of "unsigned bytes" of the given
+    length. Returns 0 on success and -1 (with raising an error) on error.
+    """
+    flags = rffi.cast(lltype.Signed, flags)
+    if flags & PyBUF_WRITABLE and readonly:
+        raise oefmt(space.w_ValueError, "Object is not writable")
+    view.c_buf = buf
+    view.c_len = length
+    view.c_obj = obj
+    if obj:
+        Py_IncRef(space, obj)
+    view.c_itemsize = 1
+    rffi.setintfield(view, 'c_readonly', readonly)
+    rffi.setintfield(view, 'c_ndim', 1)
+    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
+    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
+    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
+    return 0
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,13 +1,11 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function, cts,
-    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
+    cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t,
+    PyVarObject, size_t, slot_function, cts,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
-    Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
+    Py_GE, CONST_STRING, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
-    PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
-    get_typedescr, _Py_NewReference)
+    PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef, get_typedescr)
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
@@ -16,10 +14,6 @@
 import pypy.module.__builtin__.operation as operation
 
 
-# from include/object.h
-PyBUF_SIMPLE = 0x0000
-PyBUF_WRITABLE = 0x0001
-
 @cpython_api([size_t], rffi.VOIDP)
 def PyObject_Malloc(space, size):
     # returns non-zero-initialized memory, like CPython
@@ -444,36 +438,6 @@
     is active then NULL is returned but PyErr_Occurred() will return false."""
     return space.call_function(space.builtin.get('dir'), w_o)
 
- at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
-def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
-    """Returns a pointer to a read-only memory location usable as
-    character-based input.  The obj argument must support the single-segment
-    character buffer interface.  On success, returns 0, sets buffer to the
-    memory location and size to the buffer length.  Returns -1 and sets a
-    TypeError on error.
-    """
-    pto = obj.c_ob_type
-
-    pb = pto.c_tp_as_buffer
-    if not (pb and pb.c_bf_getbuffer):
-        raise oefmt(space.w_TypeError,
-                    "expected an object with the buffer interface")
-    with lltype.scoped_alloc(Py_buffer) as view:
-        ret = generic_cpy_call(
-            space, pb.c_bf_getbuffer,
-            obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
-        if rffi.cast(lltype.Signed, ret) == -1:
-            return -1
-
-        bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
-        sizep[0] = view.c_len
-
-        if pb.c_bf_releasebuffer:
-            generic_cpy_call(space, pb.c_bf_releasebuffer,
-                             obj, view)
-        Py_DecRef(space, view.c_obj)
-    return 0
-
 # Also in include/object.h
 Py_PRINT_RAW = 1 # No string quotes etc.
 
@@ -493,41 +457,3 @@
     with rffi.scoped_nonmovingbuffer(data) as buf:
         fwrite(buf, 1, count, fp)
     return 0
-
-
-PyBUF_WRITABLE = 0x0001  # Copied from object.h
-
- at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
-              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
-def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
-    """
-    Fills in a buffer-info structure correctly for an exporter that can only
-    share a contiguous chunk of memory of "unsigned bytes" of the given
-    length. Returns 0 on success and -1 (with raising an error) on error.
-    """
-    flags = rffi.cast(lltype.Signed, flags)
-    if flags & PyBUF_WRITABLE and readonly:
-        raise oefmt(space.w_ValueError, "Object is not writable")
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    if obj:
-        Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 1)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
-        view.c_format = rffi.str2charp("B")
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_ND) == PyBUF_ND:
-        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-        view.c_shape[0] = view.c_len
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
-        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-        view.c_strides[0] = view.c_itemsize
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
-    return 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -548,14 +548,14 @@
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bytes_getbuffer(space, w_str, view, flags):
     from pypy.module.cpyext.bytesobject import PyBytes_AsString
-    from pypy.module.cpyext.object import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
     c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str))
     return PyBuffer_FillInfo(space, view, w_str, c_buf,
                              space.len_w(w_str), 1, flags)
 
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bf_getbuffer(space, w_obj, view, flags):
-    from pypy.module.cpyext.object import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
     buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
     c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     return PyBuffer_FillInfo(space, view, w_obj, c_buf,

From pypy.commits at gmail.com  Sun Feb  5 15:39:28 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 05 Feb 2017 12:39:28 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58978d80.ce9adf0a.bd348.a643@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r89955:b32bdcb0c782
Date: 2017-02-05 20:38 +0000
http://bitbucket.org/pypy/pypy/changeset/b32bdcb0c782/

Log:	hg merge default

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -258,6 +258,8 @@
         assert ffi.buffer(a)[:] == b'\x05\x06\x07'
         assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
         assert type(ffi.buffer(a)) is ffi.buffer
+        assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+        assert type(ffi.buffer(a)) is ffi.buffer
 
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend

From pypy.commits at gmail.com  Sun Feb  5 15:45:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 12:45:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix minor merge error
Message-ID: <58978ef6.ce9adf0a.bd348.a82a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89956:25ba5fb90782
Date: 2017-02-05 21:44 +0100
http://bitbucket.org/pypy/pypy/changeset/25ba5fb90782/

Log:	fix minor merge error

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -258,8 +258,6 @@
         assert ffi.buffer(a)[:] == b'\x05\x06\x07'
         assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
         assert type(ffi.buffer(a)) is ffi.buffer
-        assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
-        assert type(ffi.buffer(a)) is ffi.buffer
 
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend

From pypy.commits at gmail.com  Sun Feb  5 15:51:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 12:51:36 -0800 (PST)
Subject: [pypy-commit] pypy default: Add objectmodel.delitem_if_value_is(),
 to be used next
Message-ID: <58979058.45aadf0a.33c59.acae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89957:bfc27e0e9d42
Date: 2017-02-05 19:46 +0100
http://bitbucket.org/pypy/pypy/changeset/bfc27e0e9d42/

Log:	Add objectmodel.delitem_if_value_is(), to be used next

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -575,6 +575,10 @@
         pair(self, s_key).delitem()
     method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
 
+    def method_delitem_if_value_is(self, s_key, s_value):
+        pair(self, s_key).setitem(s_value)
+        pair(self, s_key).delitem()
+
 class __extend__(SomeOrderedDict):
 
     def method_move_to_end(self, s_key, s_last):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,20 @@
         return
     d.delitem_with_hash(key, h)
 
+ at specialize.call_location()
+def delitem_if_value_is(d, key, value):
+    """Same as 'if d.get(key) is value: del d[key]'.  It is safe even in
+    case 'd' is an r_dict and the lookup involves callbacks that might
+    release the GIL."""
+    if not we_are_translated():
+        try:
+            if d[key] is value:
+                del d[key]
+        except KeyError:
+            pass
+        return
+    d.delitem_if_value_is(key, value)
+
 def _untranslated_move_to_end(d, key, last):
     "NOT_RPYTHON"
     value = d.pop(key)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config, try_inline, move_to_end)
+    fetch_translated_config, try_inline, delitem_if_value_is, move_to_end)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -661,6 +661,24 @@
     f(29)
     interpret(f, [27])
 
+def test_delitem_if_value_is():
+    class X:
+        pass
+    def f(i):
+        x42 = X()
+        x612 = X()
+        d = {i + .5: x42, i + .6: x612}
+        delitem_if_value_is(d, i + .5, x612)
+        assert (i + .5) in d
+        delitem_if_value_is(d, i + .5, x42)
+        assert (i + .5) not in d
+        delitem_if_value_is(d, i + .5, x612)
+        assert (i + .5) not in d
+        return 0
+
+    f(29)
+    interpret(f, [27])
+
 def test_rdict_with_hash():
     def f(i):
         d = r_dict(strange_key_eq, strange_key_hash)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,12 @@
         hop.exception_is_here()
         hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
 
+    def rtype_method_delitem_if_value_is(self, hop):
+        v_dict, v_key, v_value = hop.inputargs(
+            self, self.key_repr, self.value_repr)
+        hop.exception_cannot_occur()
+        hop.gendirectcall(ll_dict_delitem_if_value_is, v_dict, v_key, v_value)
+
     def rtype_method_move_to_end(self, hop):
         v_dict, v_key, v_last = hop.inputargs(
             self, self.key_repr, lltype.Bool)
@@ -821,6 +827,15 @@
         raise KeyError
     _ll_dict_del(d, hash, index)
 
+def ll_dict_delitem_if_value_is(d, key, value):
+    hash = d.keyhash(key)
+    index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if index < 0:
+        return
+    if d.entries[index].value != value:
+        return
+    _ll_dict_del(d, hash, index)
+
 def _ll_dict_del_entry(d, index):
     d.entries.mark_deleted(index)
     d.num_live_items -= 1

From pypy.commits at gmail.com  Sun Feb  5 15:51:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 12:51:38 -0800 (PST)
Subject: [pypy-commit] pypy default: Use objectmodel.delitem_if_value_is()
 to fix lib-python/2.7/weakref.py
Message-ID: <5897905a.c53f1c0a.e9477.f757@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89958:cba1fc5e5dcf
Date: 2017-02-05 20:54 +0100
http://bitbucket.org/pypy/pypy/changeset/cba1fc5e5dcf/

Log:	Use objectmodel.delitem_if_value_is() to fix lib-
	python/2.7/weakref.py

diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -31,6 +31,26 @@
            "WeakKeyDictionary", "ReferenceError", "ReferenceType", "ProxyType",
            "CallableProxyType", "ProxyTypes", "WeakValueDictionary", 'WeakSet']
 
+try:
+    from __pypy__ import delitem_if_value_is as _delitem_if_value_is
+except ImportError:
+    def _delitem_if_value_is(d, key, value):
+        try:
+            if self.data[key] is value:  # fall-back: there is a potential
+                #             race condition in multithreaded programs HERE
+                del self.data[key]
+        except KeyError:
+            pass
+
+def _remove_dead_weakref(d, key):
+    try:
+        wr = d[key]
+    except KeyError:
+        pass
+    else:
+        if wr() is None:
+            _delitem_if_value_is(d, key, wr)
+
 
 class WeakValueDictionary(UserDict.UserDict):
     """Mapping class that references values weakly.
@@ -58,14 +78,9 @@
                 if self._iterating:
                     self._pending_removals.append(wr.key)
                 else:
-                    # Changed this for PyPy: made more resistent.  The
-                    # issue is that in some corner cases, self.data
-                    # might already be changed or removed by the time
-                    # this weakref's callback is called.  If that is
-                    # the case, we don't want to randomly kill an
-                    # unrelated entry.
-                    if self.data.get(wr.key) is wr:
-                        del self.data[wr.key]
+                    # Atomic removal is necessary since this function
+                    # can be called asynchronously by the GC
+                    _delitem_if_value_is(self.data, wr.key, wr)
         self._remove = remove
         # A list of keys to be removed
         self._pending_removals = []
@@ -78,7 +93,8 @@
         # We shouldn't encounter any KeyError, because this method should
         # always be called *before* mutating the dict.
         while l:
-            del d[l.pop()]
+            key = l.pop()
+            _remove_dead_weakref(d, key)
 
     def __getitem__(self, key):
         o = self.data[key]()
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -79,6 +79,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
+        'delitem_if_value_is'       : 'interp_dict.delitem_if_value_is',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'specialized_zip_2_lists'   : 'interp_magic.specialized_zip_2_lists',
         'set_debug'                 : 'interp_magic.set_debug',
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -44,3 +44,16 @@
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_reversed_dict(space)
+
+def delitem_if_value_is(space, w_obj, w_key, w_value):
+    """Atomic equivalent to: 'if dict.get(key) is value: del dict[key]'.
+
+    SPECIAL USE CASES ONLY!  Avoid using on dicts which are specialized,
+    e.g. to int or str keys, because it switches to the object strategy.
+    Also, the 'is' operation is really pointer equality, so avoid using
+    it if 'value' is an immutable object like int or str.
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_delitem_if_value_is(space, w_key, w_value)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -283,6 +283,14 @@
             w_keys = self.w_keys()
             return space.call_method(w_keys, '__reversed__')
 
+    def nondescr_delitem_if_value_is(self, space, w_key, w_value):
+        """Not exposed directly to app-level, but used by
+        _weakref._remove_dead_weakref and via __pypy__.delitem_if_value_is().
+        """
+        strategy = self.ensure_object_strategy()
+        d = strategy.unerase(self.dstorage)
+        objectmodel.delitem_if_value_is(d, w_key, w_value)
+
     def descr_viewitems(self, space):
         """D.viewitems() -> a set-like object providing a view on D's items"""
         return W_DictViewItemsObject(space, self)
@@ -350,11 +358,12 @@
         F: D[k] = F[k]"""
         init_or_update(space, self, __args__, 'dict.update')
 
-    def ensure_object_strategy(self):    # for cpyext
+    def ensure_object_strategy(self):    # also called by cpyext
         object_strategy = self.space.fromcache(ObjectDictStrategy)
         strategy = self.get_strategy()
         if strategy is not object_strategy:
             strategy.switch_to_object_strategy(self)
+        return object_strategy
 
 
 class W_DictObject(W_DictMultiObject):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -271,6 +271,20 @@
         del d[key]
         raises(RuntimeError, it.next)
 
+    def test_delitem_if_value_is(self):
+        import __pypy__
+        class X:
+            pass
+        x2 = X()
+        x3 = X()
+        d = {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x2)
+        assert d == {3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {3: x3}
+
     def test_keys(self):
         d = {1: 2, 3: 4}
         kys = d.keys()

From pypy.commits at gmail.com  Sun Feb  5 15:51:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 12:51:40 -0800 (PST)
Subject: [pypy-commit] pypy default: graft 6974fd5f5c47: Add
 __pypy__.move_to_end(), similar to
Message-ID: <5897905c.545a1c0a.92a70.fc40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89959:e0ea69f0a7d8
Date: 2017-02-05 21:25 +0100
http://bitbucket.org/pypy/pypy/changeset/e0ea69f0a7d8/

Log:	graft 6974fd5f5c47: Add __pypy__.move_to_end(), similar to
	__pypy__.reversed_dict().

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -80,6 +80,7 @@
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
         'delitem_if_value_is'       : 'interp_dict.delitem_if_value_is',
+        'move_to_end'               : 'interp_dict.move_to_end',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'specialized_zip_2_lists'   : 'interp_magic.specialized_zip_2_lists',
         'set_debug'                 : 'interp_magic.set_debug',
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -57,3 +57,14 @@
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_delitem_if_value_is(space, w_key, w_value)
+
+ at unwrap_spec(last=bool)
+def move_to_end(space, w_obj, w_key, last=True):
+    """Move the key in a dictionary object into the first or last position.
+
+    This is used in Python 3.x to implement OrderedDict.move_to_end().
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_move_to_end(space, w_key, last)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -307,6 +307,37 @@
         """D.has_key(k) -> True if D has a key k, else False"""
         return space.newbool(self.getitem(w_key) is not None)
 
+    def nondescr_move_to_end(self, space, w_key, last_flag):
+        """Not exposed directly to app-level, but via __pypy__.move_to_end().
+        """
+        strategy = self.get_strategy()
+        if strategy.has_move_to_end:
+            strategy.move_to_end(self, w_key, last_flag)
+        else:
+            # fall-back
+            w_value = self.getitem(w_key)
+            if w_value is None:
+                space.raise_key_error(w_key)
+            else:
+                self.delitem(w_key)
+                if last_flag:
+                    self.setitem(w_key, w_value)
+                else:
+                    # *very slow* fall-back
+                    keys_w = []
+                    values_w = []
+                    iteratorimplementation = self.iteritems()
+                    while True:
+                        w_k, w_v = iteratorimplementation.next_item()
+                        if w_k is None:
+                            break
+                        keys_w.append(w_k)
+                        values_w.append(w_v)
+                    self.clear()
+                    self.setitem(w_key, w_value)
+                    for i in range(len(keys_w)):
+                        self.setitem(keys_w[i], values_w[i])
+
     def descr_clear(self, space):
         """D.clear() -> None.  Remove all items from D."""
         self.clear()
@@ -578,7 +609,9 @@
         raise NotImplementedError
 
     has_iterreversed = False
-    # no 'getiterreversed': no default implementation available
+    has_move_to_end = False
+    # no 'getiterreversed' and no 'move_to_end': no default
+    # implementation available
 
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         iteritems = self.iteritems(w_dict)
@@ -849,6 +882,9 @@
         dictimpl.iterreversed = iterreversed
         dictimpl.has_iterreversed = True
 
+    if hasattr(dictimpl, 'move_to_end'):
+        dictimpl.has_move_to_end = True
+
     @jit.look_inside_iff(lambda self, w_dict, w_updatedict:
                          w_dict_unrolling_heuristic(w_dict))
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
@@ -1013,6 +1049,15 @@
     def getiterreversed(self, w_dict):
         return objectmodel.reversed_dict(self.unerase(w_dict.dstorage))
 
+    def move_to_end(self, w_dict, w_key, last_flag):
+        if self.is_correct_type(w_key):
+            d = self.unerase(w_dict.dstorage)
+            key = self.unwrap(w_key)
+            objectmodel.move_to_end(d, key, last_flag)
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.nondescr_move_to_end(w_dict.space, w_key, last_flag)
+
     def prepare_update(self, w_dict, num_extra):
         objectmodel.prepare_dict_update(self.unerase(w_dict.dstorage),
                                         num_extra)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -285,6 +285,29 @@
         __pypy__.delitem_if_value_is(d, 2, x3)
         assert d == {3: x3}
 
+    def test_move_to_end(self):
+        import __pypy__
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo')
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=True)
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=False)
+        def kwdict(**k):
+            return k
+        for last in [False, True]:
+            for d, key in [({1: 2, 3: 4, 5: 6}, 3),
+                           ({"a": 5, "b": 2, "c": 6}, "b"),
+                           (kwdict(d=7, e=8, f=9), "e")]:
+                other_keys = [k for k in d if k != key]
+                __pypy__.move_to_end(d, key, last=last)
+                if not self.on_pypy:
+                    # when running tests on CPython, the underlying
+                    # dicts are not ordered.  We don't get here if
+                    # we're running tests on PyPy or with -A.
+                    assert set(d.keys()) == set(other_keys + [key])
+                elif last:
+                    assert list(d) == other_keys + [key]
+                else:
+                    assert list(d) == [key] + other_keys
+
     def test_keys(self):
         d = {1: 2, 3: 4}
         kys = d.keys()

From pypy.commits at gmail.com  Sun Feb  5 15:51:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 12:51:42 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5897905e.5190df0a.5d65e.9f22@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89960:357c149fd26e
Date: 2017-02-05 21:49 +0100
http://bitbucket.org/pypy/pypy/changeset/357c149fd26e/

Log:	merge heads

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -126,7 +126,7 @@
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
-PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,7 +1,10 @@
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts)
-from pypy.module.cpyext.pyobject import PyObject
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts, Py_buffer,
+    Py_ssize_t, Py_ssize_tP,
+    PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES)
+from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -11,7 +14,52 @@
     if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
         return 1
     name = rffi.charp2str(cts.cast('char*', pyobj.c_ob_type.c_tp_name))
-    if  name in ('str', 'bytes'):
+    if name in ('str', 'bytes'):
         # XXX remove once wrapper of __buffer__ -> bf_getbuffer works
         return 1
     return 0
+
+ at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
+              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
+def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
+    """
+    Fills in a buffer-info structure correctly for an exporter that can only
+    share a contiguous chunk of memory of "unsigned bytes" of the given
+    length. Returns 0 on success and -1 (with raising an error) on error.
+    """
+    if flags & PyBUF_WRITABLE and readonly:
+        raise oefmt(space.w_ValueError, "Object is not writable")
+    view.c_buf = buf
+    view.c_len = length
+    view.c_obj = obj
+    if obj:
+        Py_IncRef(space, obj)
+    view.c_itemsize = 1
+    rffi.setintfield(view, 'c_readonly', readonly)
+    rffi.setintfield(view, 'c_ndim', 1)
+    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
+    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
+    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
+    return 0
+
+
+ at cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
+def PyBuffer_Release(space, view):
+    """
+    Release the buffer view. This should be called when the buffer is
+    no longer being used as it may free memory from it
+    """
+    Py_DecRef(space, view.c_obj)
+    view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,13 +1,12 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function,
-    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
+    PyVarObject, size_t, slot_function,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
     Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
-    PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
-    get_typedescr, _Py_NewReference)
+    PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef,
+    get_typedescr)
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
@@ -476,51 +475,3 @@
     with rffi.scoped_nonmovingbuffer(data) as buf:
         fwrite(buf, 1, count, fp)
     return 0
-
-
-PyBUF_WRITABLE = 0x0001  # Copied from object.h
-
- at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
-              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
-def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
-    """
-    Fills in a buffer-info structure correctly for an exporter that can only
-    share a contiguous chunk of memory of "unsigned bytes" of the given
-    length. Returns 0 on success and -1 (with raising an error) on error.
-    """
-    if flags & PyBUF_WRITABLE and readonly:
-        raise oefmt(space.w_ValueError, "Object is not writable")
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    if obj:
-        Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 1)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
-        view.c_format = rffi.str2charp("B")
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_ND) == PyBUF_ND:
-        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-        view.c_shape[0] = view.c_len
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
-        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-        view.c_strides[0] = view.c_itemsize
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
-    return 0
-
-
- at cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
-def PyBuffer_Release(space, view):
-    """
-    Release the buffer view. This should be called when the buffer is
-    no longer being used as it may free memory from it
-    """
-    Py_DecRef(space, view.c_obj)
-    view.c_obj = lltype.nullptr(PyObject.TO)
-    # XXX do other fields leak memory?

From pypy.commits at gmail.com  Sun Feb  5 16:03:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 13:03:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5897932b.958c1c0a.74ac5.02e3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89961:4bd2a3132a3f
Date: 2017-02-05 22:01 +0100
http://bitbucket.org/pypy/pypy/changeset/4bd2a3132a3f/

Log:	hg merge default

diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -31,6 +31,26 @@
            "WeakKeyDictionary", "ReferenceError", "ReferenceType", "ProxyType",
            "CallableProxyType", "ProxyTypes", "WeakValueDictionary", 'WeakSet']
 
+try:
+    from __pypy__ import delitem_if_value_is as _delitem_if_value_is
+except ImportError:
+    def _delitem_if_value_is(d, key, value):
+        try:
+            if self.data[key] is value:  # fall-back: there is a potential
+                #             race condition in multithreaded programs HERE
+                del self.data[key]
+        except KeyError:
+            pass
+
+def _remove_dead_weakref(d, key):
+    try:
+        wr = d[key]
+    except KeyError:
+        pass
+    else:
+        if wr() is None:
+            _delitem_if_value_is(d, key, wr)
+
 
 class WeakValueDictionary(UserDict.UserDict):
     """Mapping class that references values weakly.
@@ -58,14 +78,9 @@
                 if self._iterating:
                     self._pending_removals.append(wr.key)
                 else:
-                    # Changed this for PyPy: made more resistent.  The
-                    # issue is that in some corner cases, self.data
-                    # might already be changed or removed by the time
-                    # this weakref's callback is called.  If that is
-                    # the case, we don't want to randomly kill an
-                    # unrelated entry.
-                    if self.data.get(wr.key) is wr:
-                        del self.data[wr.key]
+                    # Atomic removal is necessary since this function
+                    # can be called asynchronously by the GC
+                    _delitem_if_value_is(self.data, wr.key, wr)
         self._remove = remove
         # A list of keys to be removed
         self._pending_removals = []
@@ -78,7 +93,8 @@
         # We shouldn't encounter any KeyError, because this method should
         # always be called *before* mutating the dict.
         while l:
-            del d[l.pop()]
+            key = l.pop()
+            _remove_dead_weakref(d, key)
 
     def __getitem__(self, key):
         o = self.data[key]()
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -78,6 +78,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
+        'delitem_if_value_is'       : 'interp_dict.delitem_if_value_is',
         'move_to_end'               : 'interp_dict.move_to_end',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -59,3 +59,16 @@
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_move_to_end(space, w_key, last)
+
+def delitem_if_value_is(space, w_obj, w_key, w_value):
+    """Atomic equivalent to: 'if dict.get(key) is value: del dict[key]'.
+
+    SPECIAL USE CASES ONLY!  Avoid using on dicts which are specialized,
+    e.g. to int or str keys, because it switches to the object strategy.
+    Also, the 'is' operation is really pointer equality, so avoid using
+    it if 'value' is an immutable object like int or str.
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_delitem_if_value_is(space, w_key, w_value)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -263,6 +263,14 @@
                     for i in range(len(keys_w)):
                         self.setitem(keys_w[i], values_w[i])
 
+    def nondescr_delitem_if_value_is(self, space, w_key, w_value):
+        """Not exposed directly to app-level, but used by
+        _weakref._remove_dead_weakref and via __pypy__.delitem_if_value_is().
+        """
+        strategy = self.ensure_object_strategy()
+        d = strategy.unerase(self.dstorage)
+        objectmodel.delitem_if_value_is(d, w_key, w_value)
+
     def descr_clear(self, space):
         """D.clear() -> None.  Remove all items from D."""
         self.clear()
@@ -314,11 +322,12 @@
         F: D[k] = F[k]"""
         init_or_update(space, self, __args__, 'dict.update')
 
-    def ensure_object_strategy(self):    # for cpyext
+    def ensure_object_strategy(self):    # also called by cpyext
         object_strategy = self.space.fromcache(ObjectDictStrategy)
         strategy = self.get_strategy()
         if strategy is not object_strategy:
             strategy.switch_to_object_strategy(self)
+        return object_strategy
 
 
 class W_DictObject(W_DictMultiObject):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -293,6 +293,20 @@
                 else:
                     assert list(d) == [key] + other_keys
 
+    def test_delitem_if_value_is(self):
+        import __pypy__
+        class X:
+            pass
+        x2 = X()
+        x3 = X()
+        d = {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x2)
+        assert d == {3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {3: x3}
+
     def test_keys(self):
         d = {1: 2, 3: 4}
         kys = list(d.keys())
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -575,6 +575,10 @@
         pair(self, s_key).delitem()
     method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
 
+    def method_delitem_if_value_is(self, s_key, s_value):
+        pair(self, s_key).setitem(s_value)
+        pair(self, s_key).delitem()
+
 class __extend__(SomeOrderedDict):
 
     def method_move_to_end(self, s_key, s_last):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,20 @@
         return
     d.delitem_with_hash(key, h)
 
+ at specialize.call_location()
+def delitem_if_value_is(d, key, value):
+    """Same as 'if d.get(key) is value: del d[key]'.  It is safe even in
+    case 'd' is an r_dict and the lookup involves callbacks that might
+    release the GIL."""
+    if not we_are_translated():
+        try:
+            if d[key] is value:
+                del d[key]
+        except KeyError:
+            pass
+        return
+    d.delitem_if_value_is(key, value)
+
 def _untranslated_move_to_end(d, key, last):
     "NOT_RPYTHON"
     value = d.pop(key)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config, try_inline, move_to_end)
+    fetch_translated_config, try_inline, delitem_if_value_is, move_to_end)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -661,6 +661,24 @@
     f(29)
     interpret(f, [27])
 
+def test_delitem_if_value_is():
+    class X:
+        pass
+    def f(i):
+        x42 = X()
+        x612 = X()
+        d = {i + .5: x42, i + .6: x612}
+        delitem_if_value_is(d, i + .5, x612)
+        assert (i + .5) in d
+        delitem_if_value_is(d, i + .5, x42)
+        assert (i + .5) not in d
+        delitem_if_value_is(d, i + .5, x612)
+        assert (i + .5) not in d
+        return 0
+
+    f(29)
+    interpret(f, [27])
+
 def test_rdict_with_hash():
     def f(i):
         d = r_dict(strange_key_eq, strange_key_hash)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,12 @@
         hop.exception_is_here()
         hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
 
+    def rtype_method_delitem_if_value_is(self, hop):
+        v_dict, v_key, v_value = hop.inputargs(
+            self, self.key_repr, self.value_repr)
+        hop.exception_cannot_occur()
+        hop.gendirectcall(ll_dict_delitem_if_value_is, v_dict, v_key, v_value)
+
     def rtype_method_move_to_end(self, hop):
         v_dict, v_key, v_last = hop.inputargs(
             self, self.key_repr, lltype.Bool)
@@ -821,6 +827,15 @@
         raise KeyError
     _ll_dict_del(d, hash, index)
 
+def ll_dict_delitem_if_value_is(d, key, value):
+    hash = d.keyhash(key)
+    index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if index < 0:
+        return
+    if d.entries[index].value != value:
+        return
+    _ll_dict_del(d, hash, index)
+
 def _ll_dict_del_entry(d, index):
     d.entries.mark_deleted(index)
     d.num_live_items -= 1

From pypy.commits at gmail.com  Sun Feb  5 16:46:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 13:46:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge vendor/stdlib-3.5: upgrade
 stdlib to 3.5.3
Message-ID: <58979d2b.c53f1c0a.e9477.0840@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89962:08ae26119b9a
Date: 2017-02-05 22:45 +0100
http://bitbucket.org/pypy/pypy/changeset/08ae26119b9a/

Log:	hg merge vendor/stdlib-3.5: upgrade stdlib to 3.5.3

diff too long, truncating to 2000 out of 24845 lines

diff --git a/lib-python/3/_collections_abc.py b/lib-python/3/_collections_abc.py
--- a/lib-python/3/_collections_abc.py
+++ b/lib-python/3/_collections_abc.py
@@ -29,8 +29,8 @@
 # so that they will pass tests like:
 #       it = iter(somebytearray)
 #       assert isinstance(it, Iterable)
-# Note:  in other implementations, these types many not be distinct
-# and they make have their own implementation specific types that
+# Note:  in other implementations, these types might not be distinct
+# and they may have their own implementation specific types that
 # are not included on this list.
 bytes_iterator = type(iter(b''))
 bytearray_iterator = type(iter(bytearray()))
@@ -41,6 +41,7 @@
 list_iterator = type(iter([]))
 list_reverseiterator = type(iter(reversed([])))
 range_iterator = type(iter(range(0)))
+longrange_iterator = type(iter(range(1 << 1000)))
 set_iterator = type(iter(set()))
 str_iterator = type(iter(""))
 tuple_iterator = type(iter(()))
@@ -234,6 +235,7 @@
 Iterator.register(list_iterator)
 Iterator.register(list_reverseiterator)
 Iterator.register(range_iterator)
+Iterator.register(longrange_iterator)
 Iterator.register(set_iterator)
 Iterator.register(str_iterator)
 Iterator.register(tuple_iterator)
diff --git a/lib-python/3/_pydecimal.py b/lib-python/3/_pydecimal.py
--- a/lib-python/3/_pydecimal.py
+++ b/lib-python/3/_pydecimal.py
@@ -1068,12 +1068,11 @@
         return sign + intpart + fracpart + exp
 
     def to_eng_string(self, context=None):
-        """Convert to engineering-type string.
-
-        Engineering notation has an exponent which is a multiple of 3, so there
-        are up to 3 digits left of the decimal place.
-
-        Same rules for when in exponential and when as a value as in __str__.
+        """Convert to a string, using engineering notation if an exponent is needed.
+
+        Engineering notation has an exponent which is a multiple of 3.  This
+        can leave up to 3 digits to the left of the decimal place and may
+        require the addition of either one or two trailing zeros.
         """
         return self.__str__(eng=True, context=context)
 
@@ -4107,7 +4106,7 @@
         >>> context.create_decimal_from_float(3.1415926535897932)
         Traceback (most recent call last):
             ...
-        decimal.Inexact
+        decimal.Inexact: None
 
         """
         d = Decimal.from_float(f)       # An exact conversion
@@ -5502,9 +5501,29 @@
             return r
 
     def to_eng_string(self, a):
-        """Converts a number to a string, using scientific notation.
+        """Convert to a string, using engineering notation if an exponent is needed.
+
+        Engineering notation has an exponent which is a multiple of 3.  This
+        can leave up to 3 digits to the left of the decimal place and may
+        require the addition of either one or two trailing zeros.
 
         The operation is not affected by the context.
+
+        >>> ExtendedContext.to_eng_string(Decimal('123E+1'))
+        '1.23E+3'
+        >>> ExtendedContext.to_eng_string(Decimal('123E+3'))
+        '123E+3'
+        >>> ExtendedContext.to_eng_string(Decimal('123E-10'))
+        '12.3E-9'
+        >>> ExtendedContext.to_eng_string(Decimal('-123E-12'))
+        '-123E-12'
+        >>> ExtendedContext.to_eng_string(Decimal('7E-7'))
+        '700E-9'
+        >>> ExtendedContext.to_eng_string(Decimal('7E+1'))
+        '70'
+        >>> ExtendedContext.to_eng_string(Decimal('0E+1'))
+        '0.00E+3'
+
         """
         a = _convert_other(a, raiseit=True)
         return a.to_eng_string(context=self)
diff --git a/lib-python/3/_pyio.py b/lib-python/3/_pyio.py
--- a/lib-python/3/_pyio.py
+++ b/lib-python/3/_pyio.py
@@ -276,7 +276,7 @@
 try:
     UnsupportedOperation = io.UnsupportedOperation
 except AttributeError:
-    class UnsupportedOperation(ValueError, OSError):
+    class UnsupportedOperation(OSError, ValueError):
         pass
 
 
diff --git a/lib-python/3/antigravity.py b/lib-python/3/antigravity.py
--- a/lib-python/3/antigravity.py
+++ b/lib-python/3/antigravity.py
@@ -2,7 +2,7 @@
 import webbrowser
 import hashlib
 
-webbrowser.open("http://xkcd.com/353/")
+webbrowser.open("https://xkcd.com/353/")
 
 def geohash(latitude, longitude, datedow):
     '''Compute geohash() using the Munroe algorithm.
diff --git a/lib-python/3/asyncio/base_events.py b/lib-python/3/asyncio/base_events.py
--- a/lib-python/3/asyncio/base_events.py
+++ b/lib-python/3/asyncio/base_events.py
@@ -13,7 +13,6 @@
 to modify the meaning of the API call itself.
 """
 
-
 import collections
 import concurrent.futures
 import heapq
@@ -28,6 +27,7 @@
 import traceback
 import sys
 import warnings
+import weakref
 
 from . import compat
 from . import coroutines
@@ -41,9 +41,6 @@
 __all__ = ['BaseEventLoop']
 
 
-# Argument for default thread pool executor creation.
-_MAX_WORKERS = 5
-
 # Minimum number of _scheduled timer handles before cleanup of
 # cancelled handles is performed.
 _MIN_SCHEDULED_TIMER_HANDLES = 100
@@ -76,12 +73,29 @@
         return repr(fd)
 
 
-# Linux's sock.type is a bitmask that can include extra info about socket.
-_SOCKET_TYPE_MASK = 0
-if hasattr(socket, 'SOCK_NONBLOCK'):
-    _SOCKET_TYPE_MASK |= socket.SOCK_NONBLOCK
-if hasattr(socket, 'SOCK_CLOEXEC'):
-    _SOCKET_TYPE_MASK |= socket.SOCK_CLOEXEC
+def _set_reuseport(sock):
+    if not hasattr(socket, 'SO_REUSEPORT'):
+        raise ValueError('reuse_port not supported by socket module')
+    else:
+        try:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        except OSError:
+            raise ValueError('reuse_port not supported by socket module, '
+                             'SO_REUSEPORT defined but not implemented.')
+
+
+def _is_stream_socket(sock):
+    # Linux's socket.type is a bitmask that can include extra info
+    # about socket, therefore we can't do simple
+    # `sock_type == socket.SOCK_STREAM`.
+    return (sock.type & socket.SOCK_STREAM) == socket.SOCK_STREAM
+
+
+def _is_dgram_socket(sock):
+    # Linux's socket.type is a bitmask that can include extra info
+    # about socket, therefore we can't do simple
+    # `sock_type == socket.SOCK_DGRAM`.
+    return (sock.type & socket.SOCK_DGRAM) == socket.SOCK_DGRAM
 
 
 def _ipaddr_info(host, port, family, type, proto):
@@ -94,8 +108,12 @@
             host is None:
         return None
 
-    type &= ~_SOCKET_TYPE_MASK
     if type == socket.SOCK_STREAM:
+        # Linux only:
+        #    getaddrinfo() can raise when socket.type is a bit mask.
+        #    So if socket.type is a bit mask of SOCK_STREAM, and say
+        #    SOCK_NONBLOCK, we simply return None, which will trigger
+        #    a call to getaddrinfo() letting it process this request.
         proto = socket.IPPROTO_TCP
     elif type == socket.SOCK_DGRAM:
         proto = socket.IPPROTO_UDP
@@ -104,27 +122,21 @@
 
     if port is None:
         port = 0
-    elif isinstance(port, bytes):
-        if port == b'':
-            port = 0
-        else:
-            try:
-                port = int(port)
-            except ValueError:
-                # Might be a service name like b"http".
-                port = socket.getservbyname(port.decode('ascii'))
-    elif isinstance(port, str):
-        if port == '':
-            port = 0
-        else:
-            try:
-                port = int(port)
-            except ValueError:
-                # Might be a service name like "http".
-                port = socket.getservbyname(port)
+    elif isinstance(port, bytes) and port == b'':
+        port = 0
+    elif isinstance(port, str) and port == '':
+        port = 0
+    else:
+        # If port's a service name like "http", don't skip getaddrinfo.
+        try:
+            port = int(port)
+        except (TypeError, ValueError):
+            return None
 
     if family == socket.AF_UNSPEC:
-        afs = [socket.AF_INET, socket.AF_INET6]
+        afs = [socket.AF_INET]
+        if hasattr(socket, 'AF_INET6'):
+            afs.append(socket.AF_INET6)
     else:
         afs = [family]
 
@@ -242,6 +254,17 @@
         self._task_factory = None
         self._coroutine_wrapper_set = False
 
+        if hasattr(sys, 'get_asyncgen_hooks'):
+            # Python >= 3.6
+            # A weak set of all asynchronous generators that are
+            # being iterated by the loop.
+            self._asyncgens = weakref.WeakSet()
+        else:
+            self._asyncgens = None
+
+        # Set to True when `loop.shutdown_asyncgens` is called.
+        self._asyncgens_shutdown_called = False
+
     def __repr__(self):
         return ('<%s running=%s closed=%s debug=%s>'
                 % (self.__class__.__name__, self.is_running(),
@@ -333,14 +356,67 @@
         if self._closed:
             raise RuntimeError('Event loop is closed')
 
+    def _asyncgen_finalizer_hook(self, agen):
+        self._asyncgens.discard(agen)
+        if not self.is_closed():
+            self.create_task(agen.aclose())
+            # Wake up the loop if the finalizer was called from
+            # a different thread.
+            self._write_to_self()
+
+    def _asyncgen_firstiter_hook(self, agen):
+        if self._asyncgens_shutdown_called:
+            warnings.warn(
+                "asynchronous generator {!r} was scheduled after "
+                "loop.shutdown_asyncgens() call".format(agen),
+                ResourceWarning, source=self)
+
+        self._asyncgens.add(agen)
+
+    @coroutine
+    def shutdown_asyncgens(self):
+        """Shutdown all active asynchronous generators."""
+        self._asyncgens_shutdown_called = True
+
+        if self._asyncgens is None or not len(self._asyncgens):
+            # If Python version is <3.6 or we don't have any asynchronous
+            # generators alive.
+            return
+
+        closing_agens = list(self._asyncgens)
+        self._asyncgens.clear()
+
+        shutdown_coro = tasks.gather(
+            *[ag.aclose() for ag in closing_agens],
+            return_exceptions=True,
+            loop=self)
+
+        results = yield from shutdown_coro
+        for result, agen in zip(results, closing_agens):
+            if isinstance(result, Exception):
+                self.call_exception_handler({
+                    'message': 'an error occurred during closing of '
+                               'asynchronous generator {!r}'.format(agen),
+                    'exception': result,
+                    'asyncgen': agen
+                })
+
     def run_forever(self):
         """Run until stop() is called."""
         self._check_closed()
         if self.is_running():
-            raise RuntimeError('Event loop is running.')
+            raise RuntimeError('This event loop is already running')
+        if events._get_running_loop() is not None:
+            raise RuntimeError(
+                'Cannot run the event loop while another loop is running')
         self._set_coroutine_wrapper(self._debug)
         self._thread_id = threading.get_ident()
+        if self._asyncgens is not None:
+            old_agen_hooks = sys.get_asyncgen_hooks()
+            sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook,
+                                   finalizer=self._asyncgen_finalizer_hook)
         try:
+            events._set_running_loop(self)
             while True:
                 self._run_once()
                 if self._stopping:
@@ -348,7 +424,10 @@
         finally:
             self._stopping = False
             self._thread_id = None
+            events._set_running_loop(None)
             self._set_coroutine_wrapper(False)
+            if self._asyncgens is not None:
+                sys.set_asyncgen_hooks(*old_agen_hooks)
 
     def run_until_complete(self, future):
         """Run until the Future is done.
@@ -363,7 +442,7 @@
         """
         self._check_closed()
 
-        new_task = not isinstance(future, futures.Future)
+        new_task = not futures.isfuture(future)
         future = tasks.ensure_future(future, loop=self)
         if new_task:
             # An exception is raised if the future didn't complete, so there
@@ -469,12 +548,10 @@
 
         Absolute time corresponds to the event loop's time() method.
         """
-        if (coroutines.iscoroutine(callback)
-        or coroutines.iscoroutinefunction(callback)):
-            raise TypeError("coroutines cannot be used with call_at()")
         self._check_closed()
         if self._debug:
             self._check_thread()
+            self._check_callback(callback, 'call_at')
         timer = events.TimerHandle(when, callback, args, self)
         if timer._source_traceback:
             del timer._source_traceback[-1]
@@ -492,18 +569,27 @@
         Any positional arguments after the callback will be passed to
         the callback when it is called.
         """
+        self._check_closed()
         if self._debug:
             self._check_thread()
+            self._check_callback(callback, 'call_soon')
         handle = self._call_soon(callback, args)
         if handle._source_traceback:
             del handle._source_traceback[-1]
         return handle
 
+    def _check_callback(self, callback, method):
+        if (coroutines.iscoroutine(callback) or
+                coroutines.iscoroutinefunction(callback)):
+            raise TypeError(
+                "coroutines cannot be used with {}()".format(method))
+        if not callable(callback):
+            raise TypeError(
+                'a callable object was expected by {}(), got {!r}'.format(
+                    method, callback))
+
+
     def _call_soon(self, callback, args):
-        if (coroutines.iscoroutine(callback)
-        or coroutines.iscoroutinefunction(callback)):
-            raise TypeError("coroutines cannot be used with call_soon()")
-        self._check_closed()
         handle = events.Handle(callback, args, self)
         if handle._source_traceback:
             del handle._source_traceback[-1]
@@ -529,6 +615,9 @@
 
     def call_soon_threadsafe(self, callback, *args):
         """Like call_soon(), but thread-safe."""
+        self._check_closed()
+        if self._debug:
+            self._check_callback(callback, 'call_soon_threadsafe')
         handle = self._call_soon(callback, args)
         if handle._source_traceback:
             del handle._source_traceback[-1]
@@ -536,22 +625,13 @@
         return handle
 
     def run_in_executor(self, executor, func, *args):
-        if (coroutines.iscoroutine(func)
-        or coroutines.iscoroutinefunction(func)):
-            raise TypeError("coroutines cannot be used with run_in_executor()")
         self._check_closed()
-        if isinstance(func, events.Handle):
-            assert not args
-            assert not isinstance(func, events.TimerHandle)
-            if func._cancelled:
-                f = self.create_future()
-                f.set_result(None)
-                return f
-            func, args = func._callback, func._args
+        if self._debug:
+            self._check_callback(func, 'run_in_executor')
         if executor is None:
             executor = self._default_executor
             if executor is None:
-                executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS)
+                executor = concurrent.futures.ThreadPoolExecutor()
                 self._default_executor = executor
         return futures.wrap_future(executor.submit(func, *args), loop=self)
 
@@ -703,11 +783,19 @@
                     raise OSError('Multiple exceptions: {}'.format(
                         ', '.join(str(exc) for exc in exceptions)))
 
-        elif sock is None:
-            raise ValueError(
-                'host and port was not specified and no sock specified')
-
-        sock.setblocking(False)
+        else:
+            if sock is None:
+                raise ValueError(
+                    'host and port was not specified and no sock specified')
+            if not _is_stream_socket(sock):
+                # We allow AF_INET, AF_INET6, AF_UNIX as long as they
+                # are SOCK_STREAM.
+                # We support passing AF_UNIX sockets even though we have
+                # a dedicated API for that: create_unix_connection.
+                # Disallowing AF_UNIX in this method, breaks backwards
+                # compatibility.
+                raise ValueError(
+                    'A Stream Socket was expected, got {!r}'.format(sock))
 
         transport, protocol = yield from self._create_connection_transport(
             sock, protocol_factory, ssl, server_hostname)
@@ -721,14 +809,17 @@
 
     @coroutine
     def _create_connection_transport(self, sock, protocol_factory, ssl,
-                                     server_hostname):
+                                     server_hostname, server_side=False):
+
+        sock.setblocking(False)
+
         protocol = protocol_factory()
         waiter = self.create_future()
         if ssl:
             sslcontext = None if isinstance(ssl, bool) else ssl
             transport = self._make_ssl_transport(
                 sock, protocol, sslcontext, waiter,
-                server_side=False, server_hostname=server_hostname)
+                server_side=server_side, server_hostname=server_hostname)
         else:
             transport = self._make_socket_transport(sock, protocol, waiter)
 
@@ -748,6 +839,9 @@
                                  allow_broadcast=None, sock=None):
         """Create datagram connection."""
         if sock is not None:
+            if not _is_dgram_socket(sock):
+                raise ValueError(
+                    'A UDP Socket was expected, got {!r}'.format(sock))
             if (local_addr or remote_addr or
                     family or proto or flags or
                     reuse_address or reuse_port or allow_broadcast):
@@ -813,12 +907,7 @@
                         sock.setsockopt(
                             socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                     if reuse_port:
-                        if not hasattr(socket, 'SO_REUSEPORT'):
-                            raise ValueError(
-                                'reuse_port not supported by socket module')
-                        else:
-                            sock.setsockopt(
-                                socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+                        _set_reuseport(sock)
                     if allow_broadcast:
                         sock.setsockopt(
                             socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
@@ -941,12 +1030,7 @@
                         sock.setsockopt(
                             socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
                     if reuse_port:
-                        if not hasattr(socket, 'SO_REUSEPORT'):
-                            raise ValueError(
-                                'reuse_port not supported by socket module')
-                        else:
-                            sock.setsockopt(
-                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
+                        _set_reuseport(sock)
                     # Disable IPv4/IPv6 dual stack support (enabled by
                     # default on Linux) which makes a single socket
                     # listen on both address families.
@@ -968,18 +1052,44 @@
         else:
             if sock is None:
                 raise ValueError('Neither host/port nor sock were specified')
+            if not _is_stream_socket(sock):
+                raise ValueError(
+                    'A Stream Socket was expected, got {!r}'.format(sock))
             sockets = [sock]
 
         server = Server(self, sockets)
         for sock in sockets:
             sock.listen(backlog)
             sock.setblocking(False)
-            self._start_serving(protocol_factory, sock, ssl, server)
+            self._start_serving(protocol_factory, sock, ssl, server, backlog)
         if self._debug:
             logger.info("%r is serving", server)
         return server
 
     @coroutine
+    def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None):
+        """Handle an accepted connection.
+
+        This is used by servers that accept connections outside of
+        asyncio but that use asyncio to handle connections.
+
+        This method is a coroutine.  When completed, the coroutine
+        returns a (transport, protocol) pair.
+        """
+        if not _is_stream_socket(sock):
+            raise ValueError(
+                'A Stream Socket was expected, got {!r}'.format(sock))
+
+        transport, protocol = yield from self._create_connection_transport(
+            sock, protocol_factory, ssl, '', server_side=True)
+        if self._debug:
+            # Get the socket from the transport because SSL transport closes
+            # the old socket and creates a new SSL socket
+            sock = transport.get_extra_info('socket')
+            logger.debug("%r handled: (%r, %r)", sock, transport, protocol)
+        return transport, protocol
+
+    @coroutine
     def connect_read_pipe(self, protocol_factory, pipe):
         protocol = protocol_factory()
         waiter = self.create_future()
@@ -1048,7 +1158,7 @@
         transport = yield from self._make_subprocess_transport(
             protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
         if self._debug:
-            logger.info('%s: %r' % (debug_log, transport))
+            logger.info('%s: %r', debug_log, transport)
         return transport, protocol
 
     @coroutine
@@ -1078,7 +1188,7 @@
             protocol, popen_args, False, stdin, stdout, stderr,
             bufsize, **kwargs)
         if self._debug:
-            logger.info('%s: %r' % (debug_log, transport))
+            logger.info('%s: %r', debug_log, transport)
         return transport, protocol
 
     def get_exception_handler(self):
@@ -1158,7 +1268,9 @@
         - 'handle' (optional): Handle instance;
         - 'protocol' (optional): Protocol instance;
         - 'transport' (optional): Transport instance;
-        - 'socket' (optional): Socket instance.
+        - 'socket' (optional): Socket instance;
+        - 'asyncgen' (optional): Asynchronous generator that caused
+                                 the exception.
 
         New keys maybe introduced in the future.
 
diff --git a/lib-python/3/asyncio/base_subprocess.py b/lib-python/3/asyncio/base_subprocess.py
--- a/lib-python/3/asyncio/base_subprocess.py
+++ b/lib-python/3/asyncio/base_subprocess.py
@@ -3,7 +3,6 @@
 import warnings
 
 from . import compat
-from . import futures
 from . import protocols
 from . import transports
 from .coroutines import coroutine
@@ -87,6 +86,12 @@
     def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
         raise NotImplementedError
 
+    def set_protocol(self, protocol):
+        self._protocol = protocol
+
+    def get_protocol(self):
+        return self._protocol
+
     def is_closing(self):
         return self._closed
 
diff --git a/lib-python/3/asyncio/coroutines.py b/lib-python/3/asyncio/coroutines.py
--- a/lib-python/3/asyncio/coroutines.py
+++ b/lib-python/3/asyncio/coroutines.py
@@ -33,12 +33,16 @@
 
 try:
     _types_coroutine = types.coroutine
+    _types_CoroutineType = types.CoroutineType
 except AttributeError:
+    # Python 3.4
     _types_coroutine = None
+    _types_CoroutineType = None
 
 try:
     _inspect_iscoroutinefunction = inspect.iscoroutinefunction
 except AttributeError:
+    # Python 3.4
     _inspect_iscoroutinefunction = lambda func: False
 
 try:
@@ -120,8 +124,8 @@
         def send(self, value):
             return self.gen.send(value)
 
-    def throw(self, exc):
-        return self.gen.throw(exc)
+    def throw(self, type, value=None, traceback=None):
+        return self.gen.throw(type, value, traceback)
 
     def close(self):
         return self.gen.close()
@@ -204,8 +208,8 @@
         @functools.wraps(func)
         def coro(*args, **kw):
             res = func(*args, **kw)
-            if isinstance(res, futures.Future) or inspect.isgenerator(res) or \
-                    isinstance(res, CoroWrapper):
+            if (futures.isfuture(res) or inspect.isgenerator(res) or
+                isinstance(res, CoroWrapper)):
                 res = yield from res
             elif _AwaitableABC is not None:
                 # If 'func' returns an Awaitable (new in 3.5) we
@@ -238,19 +242,27 @@
             w.__qualname__ = getattr(func, '__qualname__', None)
             return w
 
-    wrapper._is_coroutine = True  # For iscoroutinefunction().
+    wrapper._is_coroutine = _is_coroutine  # For iscoroutinefunction().
     return wrapper
 
 
+# A marker for iscoroutinefunction.
+_is_coroutine = object()
+
+
 def iscoroutinefunction(func):
     """Return True if func is a decorated coroutine function."""
-    return (getattr(func, '_is_coroutine', False) or
+    return (getattr(func, '_is_coroutine', None) is _is_coroutine or
             _inspect_iscoroutinefunction(func))
 
 
 _COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)
 if _CoroutineABC is not None:
     _COROUTINE_TYPES += (_CoroutineABC,)
+if _types_CoroutineType is not None:
+    # Prioritize native coroutine check to speed-up
+    # asyncio.iscoroutine.
+    _COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES
 
 
 def iscoroutine(obj):
@@ -261,6 +273,29 @@
 def _format_coroutine(coro):
     assert iscoroutine(coro)
 
+    if not hasattr(coro, 'cr_code') and not hasattr(coro, 'gi_code'):
+        # Most likely a built-in type or a Cython coroutine.
+
+        # Built-in types might not have __qualname__ or __name__.
+        coro_name = getattr(
+            coro, '__qualname__',
+            getattr(coro, '__name__', type(coro).__name__))
+        coro_name = '{}()'.format(coro_name)
+
+        running = False
+        try:
+            running = coro.cr_running
+        except AttributeError:
+            try:
+                running = coro.gi_running
+            except AttributeError:
+                pass
+
+        if running:
+            return '{} running'.format(coro_name)
+        else:
+            return coro_name
+
     coro_name = None
     if isinstance(coro, CoroWrapper):
         func = coro.func
@@ -271,7 +306,7 @@
         func = coro
 
     if coro_name is None:
-        coro_name = events._format_callback(func, ())
+        coro_name = events._format_callback(func, (), {})
 
     try:
         coro_code = coro.gi_code
diff --git a/lib-python/3/asyncio/events.py b/lib-python/3/asyncio/events.py
--- a/lib-python/3/asyncio/events.py
+++ b/lib-python/3/asyncio/events.py
@@ -6,6 +6,7 @@
            'get_event_loop_policy', 'set_event_loop_policy',
            'get_event_loop', 'set_event_loop', 'new_event_loop',
            'get_child_watcher', 'set_child_watcher',
+           '_set_running_loop', '_get_running_loop',
            ]
 
 import functools
@@ -35,23 +36,25 @@
     return None
 
 
-def _format_args(args):
-    """Format function arguments.
+def _format_args_and_kwargs(args, kwargs):
+    """Format function arguments and keyword arguments.
 
     Special case for a single parameter: ('hello',) is formatted as ('hello').
     """
     # use reprlib to limit the length of the output
-    args_repr = reprlib.repr(args)
-    if len(args) == 1 and args_repr.endswith(',)'):
-        args_repr = args_repr[:-2] + ')'
-    return args_repr
+    items = []
+    if args:
+        items.extend(reprlib.repr(arg) for arg in args)
+    if kwargs:
+        items.extend('{}={}'.format(k, reprlib.repr(v))
+                     for k, v in kwargs.items())
+    return '(' + ', '.join(items) + ')'
 
 
-def _format_callback(func, args, suffix=''):
+def _format_callback(func, args, kwargs, suffix=''):
     if isinstance(func, functools.partial):
-        if args is not None:
-            suffix = _format_args(args) + suffix
-        return _format_callback(func.func, func.args, suffix)
+        suffix = _format_args_and_kwargs(args, kwargs) + suffix
+        return _format_callback(func.func, func.args, func.keywords, suffix)
 
     if hasattr(func, '__qualname__'):
         func_repr = getattr(func, '__qualname__')
@@ -60,14 +63,13 @@
     else:
         func_repr = repr(func)
 
-    if args is not None:
-        func_repr += _format_args(args)
+    func_repr += _format_args_and_kwargs(args, kwargs)
     if suffix:
         func_repr += suffix
     return func_repr
 
 def _format_callback_source(func, args):
-    func_repr = _format_callback(func, args)
+    func_repr = _format_callback(func, args, None)
     source = _get_function_source(func)
     if source:
         func_repr += ' at %s:%s' % source
@@ -81,7 +83,6 @@
                  '_source_traceback', '_repr', '__weakref__')
 
     def __init__(self, callback, args, loop):
-        assert not isinstance(callback, Handle), 'A Handle is not a callback'
         self._loop = loop
         self._callback = callback
         self._args = args
@@ -248,6 +249,10 @@
         """
         raise NotImplementedError
 
+    def shutdown_asyncgens(self):
+        """Shutdown all active asynchronous generators."""
+        raise NotImplementedError
+
     # Methods scheduling callbacks.  All these return Handles.
 
     def _timer_handle_cancelled(self, handle):
@@ -603,6 +608,30 @@
 _lock = threading.Lock()
 
 
+# A TLS for the running event loop, used by _get_running_loop.
+class _RunningLoop(threading.local):
+    _loop = None
+_running_loop = _RunningLoop()
+
+
+def _get_running_loop():
+    """Return the running event loop or None.
+
+    This is a low-level function intended to be used by event loops.
+    This function is thread-specific.
+    """
+    return _running_loop._loop
+
+
+def _set_running_loop(loop):
+    """Set the running event loop.
+
+    This is a low-level function intended to be used by event loops.
+    This function is thread-specific.
+    """
+    _running_loop._loop = loop
+
+
 def _init_event_loop_policy():
     global _event_loop_policy
     with _lock:
@@ -628,7 +657,17 @@
 
 
 def get_event_loop():
-    """Equivalent to calling get_event_loop_policy().get_event_loop()."""
+    """Return an asyncio event loop.
+
+    When called from a coroutine or a callback (e.g. scheduled with call_soon
+    or similar API), this function will always return the running event loop.
+
+    If there is no running event loop set, the function will return
+    the result of `get_event_loop_policy().get_event_loop()` call.
+    """
+    current_loop = _get_running_loop()
+    if current_loop is not None:
+        return current_loop
     return get_event_loop_policy().get_event_loop()
 
 
diff --git a/lib-python/3/asyncio/futures.py b/lib-python/3/asyncio/futures.py
--- a/lib-python/3/asyncio/futures.py
+++ b/lib-python/3/asyncio/futures.py
@@ -2,7 +2,7 @@
 
 __all__ = ['CancelledError', 'TimeoutError',
            'InvalidStateError',
-           'Future', 'wrap_future',
+           'Future', 'wrap_future', 'isfuture',
            ]
 
 import concurrent.futures._base
@@ -110,6 +110,17 @@
             self.loop.call_exception_handler({'message': msg})
 
 
+def isfuture(obj):
+    """Check for a Future.
+
+    This returns True when obj is a Future instance or is advertising
+    itself as duck-type compatible by setting _asyncio_future_blocking.
+    See comment in Future for more details.
+    """
+    return (hasattr(obj.__class__, '_asyncio_future_blocking') and
+            obj._asyncio_future_blocking is not None)
+
+
 class Future:
     """This class is *almost* compatible with concurrent.futures.Future.
 
@@ -134,7 +145,15 @@
     _loop = None
     _source_traceback = None
 
-    _blocking = False  # proper use of future (yield vs yield from)
+    # This field is used for a dual purpose:
+    # - Its presence is a marker to declare that a class implements
+    #   the Future protocol (i.e. is intended to be duck-type compatible).
+    #   The value must also be not-None, to enable a subclass to declare
+    #   that it is not compatible by setting this to None.
+    # - It is set by __iter__() below so that Task._step() can tell
+    #   the difference between `yield from Future()` (correct) vs.
+    #   `yield Future()` (incorrect).
+    _asyncio_future_blocking = False
 
     _log_traceback = False   # Used for Python 3.4 and later
     _tb_logger = None        # Used for Python 3.3 only
@@ -357,7 +376,7 @@
 
     def __iter__(self):
         if not self.done():
-            self._blocking = True
+            self._asyncio_future_blocking = True
             yield self  # This tells Task to wait for completion.
         assert self.done(), "yield from wasn't used with future"
         return self.result()  # May raise too.
@@ -415,15 +434,17 @@
     If destination is cancelled, source gets cancelled too.
     Compatible with both asyncio.Future and concurrent.futures.Future.
     """
-    if not isinstance(source, (Future, concurrent.futures.Future)):
+    if not isfuture(source) and not isinstance(source,
+                                               concurrent.futures.Future):
         raise TypeError('A future is required for source argument')
-    if not isinstance(destination, (Future, concurrent.futures.Future)):
+    if not isfuture(destination) and not isinstance(destination,
+                                                    concurrent.futures.Future):
         raise TypeError('A future is required for destination argument')
-    source_loop = source._loop if isinstance(source, Future) else None
-    dest_loop = destination._loop if isinstance(destination, Future) else None
+    source_loop = source._loop if isfuture(source) else None
+    dest_loop = destination._loop if isfuture(destination) else None
 
     def _set_state(future, other):
-        if isinstance(future, Future):
+        if isfuture(future):
             _copy_future_state(other, future)
         else:
             _set_concurrent_future_state(future, other)
@@ -447,7 +468,7 @@
 
 def wrap_future(future, *, loop=None):
     """Wrap concurrent.futures.Future object."""
-    if isinstance(future, Future):
+    if isfuture(future):
         return future
     assert isinstance(future, concurrent.futures.Future), \
         'concurrent.futures.Future is expected, got {!r}'.format(future)
diff --git a/lib-python/3/asyncio/locks.py b/lib-python/3/asyncio/locks.py
--- a/lib-python/3/asyncio/locks.py
+++ b/lib-python/3/asyncio/locks.py
@@ -166,7 +166,7 @@
         This method blocks until the lock is unlocked, then sets it to
         locked and returns True.
         """
-        if not self._waiters and not self._locked:
+        if not self._locked and all(w.cancelled() for w in self._waiters):
             self._locked = True
             return True
 
diff --git a/lib-python/3/asyncio/proactor_events.py b/lib-python/3/asyncio/proactor_events.py
--- a/lib-python/3/asyncio/proactor_events.py
+++ b/lib-python/3/asyncio/proactor_events.py
@@ -66,6 +66,12 @@
     def _set_extra(self, sock):
         self._extra['pipe'] = sock
 
+    def set_protocol(self, protocol):
+        self._protocol = protocol
+
+    def get_protocol(self):
+        return self._protocol
+
     def is_closing(self):
         return self._closing
 
@@ -488,7 +494,7 @@
         self._csock.send(b'\0')
 
     def _start_serving(self, protocol_factory, sock,
-                       sslcontext=None, server=None):
+                       sslcontext=None, server=None, backlog=100):
 
         def loop(f=None):
             try:
diff --git a/lib-python/3/asyncio/queues.py b/lib-python/3/asyncio/queues.py
--- a/lib-python/3/asyncio/queues.py
+++ b/lib-python/3/asyncio/queues.py
@@ -7,7 +7,6 @@
 
 from . import compat
 from . import events
-from . import futures
 from . import locks
 from .coroutines import coroutine
 
diff --git a/lib-python/3/asyncio/selector_events.py b/lib-python/3/asyncio/selector_events.py
--- a/lib-python/3/asyncio/selector_events.py
+++ b/lib-python/3/asyncio/selector_events.py
@@ -11,6 +11,7 @@
 import functools
 import socket
 import warnings
+import weakref
 try:
     import ssl
 except ImportError:  # pragma: no cover
@@ -39,6 +40,17 @@
         return bool(key.events & event)
 
 
+if hasattr(socket, 'TCP_NODELAY'):
+    def _set_nodelay(sock):
+        if (sock.family in {socket.AF_INET, socket.AF_INET6} and
+                sock.type == socket.SOCK_STREAM and
+                sock.proto == socket.IPPROTO_TCP):
+            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+else:
+    def _set_nodelay(sock):
+        pass
+
+
 class BaseSelectorEventLoop(base_events.BaseEventLoop):
     """Selector event loop.
 
@@ -53,6 +65,7 @@
         logger.debug('Using selector: %s', selector.__class__.__name__)
         self._selector = selector
         self._make_self_pipe()
+        self._transports = weakref.WeakValueDictionary()
 
     def _make_socket_transport(self, sock, protocol, waiter=None, *,
                                extra=None, server=None):
@@ -104,7 +117,7 @@
         raise NotImplementedError
 
     def _close_self_pipe(self):
-        self.remove_reader(self._ssock.fileno())
+        self._remove_reader(self._ssock.fileno())
         self._ssock.close()
         self._ssock = None
         self._csock.close()
@@ -117,7 +130,7 @@
         self._ssock.setblocking(False)
         self._csock.setblocking(False)
         self._internal_fds += 1
-        self.add_reader(self._ssock.fileno(), self._read_from_self)
+        self._add_reader(self._ssock.fileno(), self._read_from_self)
 
     def _process_self_data(self, data):
         pass
@@ -151,43 +164,50 @@
                                  exc_info=True)
 
     def _start_serving(self, protocol_factory, sock,
-                       sslcontext=None, server=None):
-        self.add_reader(sock.fileno(), self._accept_connection,
-                        protocol_factory, sock, sslcontext, server)
+                       sslcontext=None, server=None, backlog=100):
+        self._add_reader(sock.fileno(), self._accept_connection,
+                         protocol_factory, sock, sslcontext, server, backlog)
 
     def _accept_connection(self, protocol_factory, sock,
-                           sslcontext=None, server=None):
-        try:
-            conn, addr = sock.accept()
-            if self._debug:
-                logger.debug("%r got a new connection from %r: %r",
-                             server, addr, conn)
-            conn.setblocking(False)
-        except (BlockingIOError, InterruptedError, ConnectionAbortedError):
-            pass  # False alarm.
-        except OSError as exc:
-            # There's nowhere to send the error, so just log it.
-            if exc.errno in (errno.EMFILE, errno.ENFILE,
-                             errno.ENOBUFS, errno.ENOMEM):
-                # Some platforms (e.g. Linux keep reporting the FD as
-                # ready, so we remove the read handler temporarily.
-                # We'll try again in a while.
-                self.call_exception_handler({
-                    'message': 'socket.accept() out of system resource',
-                    'exception': exc,
-                    'socket': sock,
-                })
-                self.remove_reader(sock.fileno())
-                self.call_later(constants.ACCEPT_RETRY_DELAY,
-                                self._start_serving,
-                                protocol_factory, sock, sslcontext, server)
+                           sslcontext=None, server=None, backlog=100):
+        # This method is only called once for each event loop tick where the
+        # listening socket has triggered an EVENT_READ. There may be multiple
+        # connections waiting for an .accept() so it is called in a loop.
+        # See https://bugs.python.org/issue27906 for more details.
+        for _ in range(backlog):
+            try:
+                conn, addr = sock.accept()
+                if self._debug:
+                    logger.debug("%r got a new connection from %r: %r",
+                                 server, addr, conn)
+                conn.setblocking(False)
+            except (BlockingIOError, InterruptedError, ConnectionAbortedError):
+                # Early exit because the socket accept buffer is empty.
+                return None
+            except OSError as exc:
+                # There's nowhere to send the error, so just log it.
+                if exc.errno in (errno.EMFILE, errno.ENFILE,
+                                 errno.ENOBUFS, errno.ENOMEM):
+                    # Some platforms (e.g. Linux keep reporting the FD as
+                    # ready, so we remove the read handler temporarily.
+                    # We'll try again in a while.
+                    self.call_exception_handler({
+                        'message': 'socket.accept() out of system resource',
+                        'exception': exc,
+                        'socket': sock,
+                    })
+                    self._remove_reader(sock.fileno())
+                    self.call_later(constants.ACCEPT_RETRY_DELAY,
+                                    self._start_serving,
+                                    protocol_factory, sock, sslcontext, server,
+                                    backlog)
+                else:
+                    raise  # The event loop will catch, log and ignore it.
             else:
-                raise  # The event loop will catch, log and ignore it.
-        else:
-            extra = {'peername': addr}
-            accept = self._accept_connection2(protocol_factory, conn, extra,
-                                              sslcontext, server)
-            self.create_task(accept)
+                extra = {'peername': addr}
+                accept = self._accept_connection2(protocol_factory, conn, extra,
+                                                  sslcontext, server)
+                self.create_task(accept)
 
     @coroutine
     def _accept_connection2(self, protocol_factory, conn, extra,
@@ -226,8 +246,18 @@
                     context['transport'] = transport
                 self.call_exception_handler(context)
 
-    def add_reader(self, fd, callback, *args):
-        """Add a reader callback."""
+    def _ensure_fd_no_transport(self, fd):
+        try:
+            transport = self._transports[fd]
+        except KeyError:
+            pass
+        else:
+            if not transport.is_closing():
+                raise RuntimeError(
+                    'File descriptor {!r} is used by transport {!r}'.format(
+                        fd, transport))
+
+    def _add_reader(self, fd, callback, *args):
         self._check_closed()
         handle = events.Handle(callback, args, self)
         try:
@@ -242,8 +272,7 @@
             if reader is not None:
                 reader.cancel()
 
-    def remove_reader(self, fd):
-        """Remove a reader callback."""
+    def _remove_reader(self, fd):
         if self.is_closed():
             return False
         try:
@@ -264,8 +293,7 @@
             else:
                 return False
 
-    def add_writer(self, fd, callback, *args):
-        """Add a writer callback.."""
+    def _add_writer(self, fd, callback, *args):
         self._check_closed()
         handle = events.Handle(callback, args, self)
         try:
@@ -280,7 +308,7 @@
             if writer is not None:
                 writer.cancel()
 
-    def remove_writer(self, fd):
+    def _remove_writer(self, fd):
         """Remove a writer callback."""
         if self.is_closed():
             return False
@@ -303,6 +331,26 @@
             else:
                 return False
 
+    def add_reader(self, fd, callback, *args):
+        """Add a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_reader(fd, callback, *args)
+
+    def remove_reader(self, fd):
+        """Remove a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_reader(fd)
+
+    def add_writer(self, fd, callback, *args):
+        """Add a writer callback.."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_writer(fd, callback, *args)
+
+    def remove_writer(self, fd):
+        """Remove a writer callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_writer(fd)
+
     def sock_recv(self, sock, n):
         """Receive data from the socket.
 
@@ -382,6 +430,7 @@
                 data = data[n:]
             self.add_writer(fd, self._sock_sendall, fut, True, sock, data)
 
+    @coroutine
     def sock_connect(self, sock, address):
         """Connect to a remote socket at address.
 
@@ -390,23 +439,16 @@
         if self._debug and sock.gettimeout() != 0:
             raise ValueError("the socket must be non-blocking")
 
+        if not hasattr(socket, 'AF_UNIX') or sock.family != socket.AF_UNIX:
+            resolved = base_events._ensure_resolved(
+                address, family=sock.family, proto=sock.proto, loop=self)
+            if not resolved.done():
+                yield from resolved
+            _, _, _, _, address = resolved.result()[0]
+
         fut = self.create_future()
-        if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX:
-            self._sock_connect(fut, sock, address)
-        else:
-            resolved = base_events._ensure_resolved(address, loop=self)
-            resolved.add_done_callback(
-                lambda resolved: self._on_resolved(fut, sock, resolved))
-
-        return fut
-
-    def _on_resolved(self, fut, sock, resolved):
-        try:
-            _, _, _, _, address = resolved.result()[0]
-        except Exception as exc:
-            fut.set_exception(exc)
-        else:
-            self._sock_connect(fut, sock, address)
+        self._sock_connect(fut, sock, address)
+        return (yield from fut)
 
     def _sock_connect(self, fut, sock, address):
         fd = sock.fileno()
@@ -417,8 +459,8 @@
             # connection runs in background. We have to wait until the socket
             # becomes writable to be notified when the connection succeed or
             # fails.
-            fut.add_done_callback(functools.partial(self._sock_connect_done,
-                                                    fd))
+            fut.add_done_callback(
+                functools.partial(self._sock_connect_done, fd))
             self.add_writer(fd, self._sock_connect_cb, fut, sock, address)
         except Exception as exc:
             fut.set_exception(exc)
@@ -482,17 +524,17 @@
             fileobj, (reader, writer) = key.fileobj, key.data
             if mask & selectors.EVENT_READ and reader is not None:
                 if reader._cancelled:
-                    self.remove_reader(fileobj)
+                    self._remove_reader(fileobj)
                 else:
                     self._add_callback(reader)
             if mask & selectors.EVENT_WRITE and writer is not None:
                 if writer._cancelled:
-                    self.remove_writer(fileobj)
+                    self._remove_writer(fileobj)
                 else:
                     self._add_callback(writer)
 
     def _stop_serving(self, sock):
-        self.remove_reader(sock.fileno())
+        self._remove_reader(sock.fileno())
         sock.close()
 
 
@@ -527,6 +569,7 @@
         self._closing = False  # Set when close() called.
         if self._server is not None:
             self._server._attach()
+        loop._transports[self._sock_fd] = self
 
     def __repr__(self):
         info = [self.__class__.__name__]
@@ -559,6 +602,12 @@
     def abort(self):
         self._force_close(None)
 
+    def set_protocol(self, protocol):
+        self._protocol = protocol
+
+    def get_protocol(self):
+        return self._protocol
+
     def is_closing(self):
         return self._closing
 
@@ -566,10 +615,10 @@
         if self._closing:
             return
         self._closing = True
-        self._loop.remove_reader(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
         if not self._buffer:
             self._conn_lost += 1
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             self._loop.call_soon(self._call_connection_lost, None)
 
     # On Python 3.3 and older, objects with a destructor part of a reference
@@ -600,10 +649,10 @@
             return
         if self._buffer:
             self._buffer.clear()
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
         if not self._closing:
             self._closing = True
-            self._loop.remove_reader(self._sock_fd)
+            self._loop._remove_reader(self._sock_fd)
         self._conn_lost += 1
         self._loop.call_soon(self._call_connection_lost, exc)
 
@@ -633,9 +682,14 @@
         self._eof = False
         self._paused = False
 
+        # Disable the Nagle algorithm -- small writes will be
+        # sent without waiting for the TCP ACK.  This generally
+        # decreases the latency (in some cases significantly.)
+        _set_nodelay(self._sock)
+
         self._loop.call_soon(self._protocol.connection_made, self)
         # only start reading when connection_made() has been called
-        self._loop.call_soon(self._loop.add_reader,
+        self._loop.call_soon(self._loop._add_reader,
                              self._sock_fd, self._read_ready)
         if waiter is not None:
             # only wake up the waiter when connection_made() has been called
@@ -648,7 +702,7 @@
         if self._paused:
             raise RuntimeError('Already paused')
         self._paused = True
-        self._loop.remove_reader(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
         if self._loop.get_debug():
             logger.debug("%r pauses reading", self)
 
@@ -658,7 +712,7 @@
         self._paused = False
         if self._closing:
             return
-        self._loop.add_reader(self._sock_fd, self._read_ready)
+        self._loop._add_reader(self._sock_fd, self._read_ready)
         if self._loop.get_debug():
             logger.debug("%r resumes reading", self)
 
@@ -682,7 +736,7 @@
                     # We're keeping the connection open so the
                     # protocol can write more, but we still can't
                     # receive more, so remove the reader callback.
-                    self._loop.remove_reader(self._sock_fd)
+                    self._loop._remove_reader(self._sock_fd)
                 else:
                     self.close()
 
@@ -715,7 +769,7 @@
                 if not data:
                     return
             # Not all was written; register write handler.
-            self._loop.add_writer(self._sock_fd, self._write_ready)
+            self._loop._add_writer(self._sock_fd, self._write_ready)
 
         # Add it to the buffer.
         self._buffer.extend(data)
@@ -731,7 +785,7 @@
         except (BlockingIOError, InterruptedError):
             pass
         except Exception as exc:
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             self._buffer.clear()
             self._fatal_error(exc, 'Fatal write error on socket transport')
         else:
@@ -739,7 +793,7 @@
                 del self._buffer[:n]
             self._maybe_resume_protocol()  # May append to buffer.
             if not self._buffer:
-                self._loop.remove_writer(self._sock_fd)
+                self._loop._remove_writer(self._sock_fd)
                 if self._closing:
                     self._call_connection_lost(None)
                 elif self._eof:
@@ -810,19 +864,19 @@
         try:
             self._sock.do_handshake()
         except ssl.SSLWantReadError:
-            self._loop.add_reader(self._sock_fd,
-                                  self._on_handshake, start_time)
+            self._loop._add_reader(self._sock_fd,
+                                   self._on_handshake, start_time)
             return
         except ssl.SSLWantWriteError:
-            self._loop.add_writer(self._sock_fd,
-                                  self._on_handshake, start_time)
+            self._loop._add_writer(self._sock_fd,
+                                   self._on_handshake, start_time)
             return
         except BaseException as exc:
             if self._loop.get_debug():
                 logger.warning("%r: SSL handshake failed",
                                self, exc_info=True)
-            self._loop.remove_reader(self._sock_fd)
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_reader(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             self._sock.close()
             self._wakeup_waiter(exc)
             if isinstance(exc, Exception):
@@ -830,8 +884,8 @@
             else:
                 raise
 
-        self._loop.remove_reader(self._sock_fd)
-        self._loop.remove_writer(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
+        self._loop._remove_writer(self._sock_fd)
 
         peercert = self._sock.getpeercert()
         if not hasattr(self._sslcontext, 'check_hostname'):
@@ -859,7 +913,7 @@
 
         self._read_wants_write = False
         self._write_wants_read = False
-        self._loop.add_reader(self._sock_fd, self._read_ready)
+        self._loop._add_reader(self._sock_fd, self._read_ready)
         self._protocol_connected = True
         self._loop.call_soon(self._protocol.connection_made, self)
         # only wake up the waiter when connection_made() has been called
@@ -881,7 +935,7 @@
         if self._paused:
             raise RuntimeError('Already paused')
         self._paused = True
-        self._loop.remove_reader(self._sock_fd)
+        self._loop._remove_reader(self._sock_fd)
         if self._loop.get_debug():
             logger.debug("%r pauses reading", self)
 
@@ -891,7 +945,7 @@
         self._paused = False
         if self._closing:
             return
-        self._loop.add_reader(self._sock_fd, self._read_ready)
+        self._loop._add_reader(self._sock_fd, self._read_ready)
         if self._loop.get_debug():
             logger.debug("%r resumes reading", self)
 
@@ -903,7 +957,7 @@
             self._write_ready()
 
             if self._buffer:
-                self._loop.add_writer(self._sock_fd, self._write_ready)
+                self._loop._add_writer(self._sock_fd, self._write_ready)
 
         try:
             data = self._sock.recv(self.max_size)
@@ -911,8 +965,8 @@
             pass
         except ssl.SSLWantWriteError:
             self._read_wants_write = True
-            self._loop.remove_reader(self._sock_fd)
-            self._loop.add_writer(self._sock_fd, self._write_ready)
+            self._loop._remove_reader(self._sock_fd)
+            self._loop._add_writer(self._sock_fd, self._write_ready)
         except Exception as exc:
             self._fatal_error(exc, 'Fatal read error on SSL transport')
         else:
@@ -937,7 +991,7 @@
             self._read_ready()
 
             if not (self._paused or self._closing):
-                self._loop.add_reader(self._sock_fd, self._read_ready)
+                self._loop._add_reader(self._sock_fd, self._read_ready)
 
         if self._buffer:
             try:
@@ -946,10 +1000,10 @@
                 n = 0
             except ssl.SSLWantReadError:
                 n = 0
-                self._loop.remove_writer(self._sock_fd)
+                self._loop._remove_writer(self._sock_fd)
                 self._write_wants_read = True
             except Exception as exc:
-                self._loop.remove_writer(self._sock_fd)
+                self._loop._remove_writer(self._sock_fd)
                 self._buffer.clear()
                 self._fatal_error(exc, 'Fatal write error on SSL transport')
                 return
@@ -960,7 +1014,7 @@
         self._maybe_resume_protocol()  # May append to buffer.
 
         if not self._buffer:
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             if self._closing:
                 self._call_connection_lost(None)
 
@@ -978,7 +1032,7 @@
             return
 
         if not self._buffer:
-            self._loop.add_writer(self._sock_fd, self._write_ready)
+            self._loop._add_writer(self._sock_fd, self._write_ready)
 
         # Add it to the buffer.
         self._buffer.extend(data)
@@ -998,7 +1052,7 @@
         self._address = address
         self._loop.call_soon(self._protocol.connection_made, self)
         # only start reading when connection_made() has been called
-        self._loop.call_soon(self._loop.add_reader,
+        self._loop.call_soon(self._loop._add_reader,
                              self._sock_fd, self._read_ready)
         if waiter is not None:
             # only wake up the waiter when connection_made() has been called
@@ -1048,7 +1102,7 @@
                     self._sock.sendto(data, addr)
                 return
             except (BlockingIOError, InterruptedError):
-                self._loop.add_writer(self._sock_fd, self._sendto_ready)
+                self._loop._add_writer(self._sock_fd, self._sendto_ready)
             except OSError as exc:
                 self._protocol.error_received(exc)
                 return
@@ -1082,6 +1136,6 @@
 
         self._maybe_resume_protocol()  # May append to buffer.
         if not self._buffer:
-            self._loop.remove_writer(self._sock_fd)
+            self._loop._remove_writer(self._sock_fd)
             if self._closing:
                 self._call_connection_lost(None)
diff --git a/lib-python/3/asyncio/sslproto.py b/lib-python/3/asyncio/sslproto.py
--- a/lib-python/3/asyncio/sslproto.py
+++ b/lib-python/3/asyncio/sslproto.py
@@ -5,6 +5,7 @@
 except ImportError:  # pragma: no cover
     ssl = None
 
+from . import base_events
 from . import compat
 from . import protocols
 from . import transports
@@ -304,6 +305,12 @@
         """Get optional transport information."""
         return self._ssl_protocol._get_extra_info(name, default)
 
+    def set_protocol(self, protocol):
+        self._app_protocol = protocol
+
+    def get_protocol(self):
+        return self._app_protocol
+
     def is_closing(self):
         return self._closed
 
@@ -403,7 +410,8 @@
     """
 
     def __init__(self, loop, app_protocol, sslcontext, waiter,
-                 server_side=False, server_hostname=None):
+                 server_side=False, server_hostname=None,
+                 call_connection_made=True):
         if ssl is None:
             raise RuntimeError('stdlib ssl module not available')
 
@@ -436,6 +444,7 @@
         self._in_shutdown = False
         # transport, ex: SelectorSocketTransport
         self._transport = None
+        self._call_connection_made = call_connection_made
 
     def _wakeup_waiter(self, exc=None):
         if self._waiter is None:
@@ -470,6 +479,7 @@
             self._loop.call_soon(self._app_protocol.connection_lost, exc)
         self._transport = None
         self._app_transport = None
+        self._wakeup_waiter(exc)
 
     def pause_writing(self):
         """Called when the low-level transport's buffer goes over
@@ -599,7 +609,8 @@
                            compression=sslobj.compression(),
                            ssl_object=sslobj,
                            )
-        self._app_protocol.connection_made(self._app_transport)
+        if self._call_connection_made:
+            self._app_protocol.connection_made(self._app_transport)
         self._wakeup_waiter()
         self._session_established = True
         # In case transport.write() was already called. Don't call
diff --git a/lib-python/3/asyncio/streams.py b/lib-python/3/asyncio/streams.py
--- a/lib-python/3/asyncio/streams.py
+++ b/lib-python/3/asyncio/streams.py
@@ -448,6 +448,7 @@
         assert not self._eof, '_wait_for_data after EOF'
 
         # Waiting for data while paused will make deadlock, so prevent it.
+        # This is essential for readexactly(n) for case when n > self._limit.
         if self._paused:
             self._paused = False
             self._transport.resume_reading()
@@ -590,7 +591,7 @@
         bytes. If the EOF was received and the internal buffer is empty, return
         an empty bytes object.
 
-        If n is zero, return empty bytes object immediatelly.
+        If n is zero, return empty bytes object immediately.
 
         If n is positive, this function try to read `n` bytes, and may return
         less or equal bytes than requested, but at least one byte. If EOF was
@@ -658,25 +659,22 @@
         if n == 0:
             return b''
 
-        # There used to be "optimized" code here.  It created its own
-        # Future and waited until self._buffer had at least the n
-        # bytes, then called read(n).  Unfortunately, this could pause
-        # the transport if the argument was larger than the pause
-        # limit (which is twice self._limit).  So now we just read()
-        # into a local buffer.
+        while len(self._buffer) < n:
+            if self._eof:
+                incomplete = bytes(self._buffer)
+                self._buffer.clear()
+                raise IncompleteReadError(incomplete, n)
 
-        blocks = []
-        while n > 0:
-            block = yield from self.read(n)
-            if not block:
-                partial = b''.join(blocks)
-                raise IncompleteReadError(partial, len(partial) + n)
-            blocks.append(block)
-            n -= len(block)
+            yield from self._wait_for_data('readexactly')
 
-        assert n == 0
-
-        return b''.join(blocks)
+        if len(self._buffer) == n:
+            data = bytes(self._buffer)
+            self._buffer.clear()
+        else:
+            data = bytes(self._buffer[:n])
+            del self._buffer[:n]
+        self._maybe_resume_transport()
+        return data
 
     if compat.PY35:
         @coroutine
diff --git a/lib-python/3/asyncio/tasks.py b/lib-python/3/asyncio/tasks.py
--- a/lib-python/3/asyncio/tasks.py
+++ b/lib-python/3/asyncio/tasks.py
@@ -241,7 +241,7 @@
                 result = coro.throw(exc)
         except StopIteration as exc:
             self.set_result(exc.value)
-        except futures.CancelledError as exc:
+        except futures.CancelledError:
             super().cancel()  # I.e., Future.cancel(self).
         except Exception as exc:
             self.set_exception(exc)
@@ -249,7 +249,8 @@
             self.set_exception(exc)
             raise
         else:
-            if isinstance(result, futures.Future):
+            blocking = getattr(result, '_asyncio_future_blocking', None)
+            if blocking is not None:
                 # Yielded Future must come from Future.__iter__().
                 if result._loop is not self._loop:
                     self._loop.call_soon(
@@ -257,13 +258,20 @@
                         RuntimeError(
                             'Task {!r} got Future {!r} attached to a '
                             'different loop'.format(self, result)))
-                elif result._blocking:
-                    result._blocking = False
-                    result.add_done_callback(self._wakeup)
-                    self._fut_waiter = result
-                    if self._must_cancel:
-                        if self._fut_waiter.cancel():
-                            self._must_cancel = False
+                elif blocking:
+                    if result is self:
+                        self._loop.call_soon(
+                            self._step,
+                            RuntimeError(
+                                'Task cannot await on itself: {!r}'.format(
+                                    self)))
+                    else:
+                        result._asyncio_future_blocking = False
+                        result.add_done_callback(self._wakeup)
+                        self._fut_waiter = result
+                        if self._must_cancel:
+                            if self._fut_waiter.cancel():
+                                self._must_cancel = False
                 else:
                     self._loop.call_soon(
                         self._step,
@@ -332,7 +340,7 @@
     Note: This does not raise TimeoutError! Futures that aren't done
     when the timeout occurs are returned in the second set.
     """
-    if isinstance(fs, futures.Future) or coroutines.iscoroutine(fs):
+    if futures.isfuture(fs) or coroutines.iscoroutine(fs):
         raise TypeError("expect a list of futures, not %s" % type(fs).__name__)
     if not fs:
         raise ValueError('Set of coroutines/Futures is empty.')
@@ -461,7 +469,7 @@
 
     Note: The futures 'f' are not necessarily members of fs.
     """
-    if isinstance(fs, futures.Future) or coroutines.iscoroutine(fs):
+    if futures.isfuture(fs) or coroutines.iscoroutine(fs):
         raise TypeError("expect a list of futures, not %s" % type(fs).__name__)
     loop = loop if loop is not None else events.get_event_loop()
     todo = {ensure_future(f, loop=loop) for f in set(fs)}
@@ -518,7 +526,7 @@
         h.cancel()
 
 
-def async(coro_or_future, *, loop=None):
+def async_(coro_or_future, *, loop=None):
     """Wrap a coroutine in a future.
 
     If the argument is a Future, it is returned directly.
@@ -531,13 +539,18 @@
 
     return ensure_future(coro_or_future, loop=loop)
 
+# Silence DeprecationWarning:
+globals()['async'] = async_
+async_.__name__ = 'async'
+del async_
+
 
 def ensure_future(coro_or_future, *, loop=None):
     """Wrap a coroutine or an awaitable in a future.
 
     If the argument is a Future, it is returned directly.
     """
-    if isinstance(coro_or_future, futures.Future):
+    if futures.isfuture(coro_or_future):
         if loop is not None and loop is not coro_or_future._loop:
             raise ValueError('loop argument must agree with Future')
         return coro_or_future
@@ -579,15 +592,21 @@
     def cancel(self):
         if self.done():
             return False
+        ret = False
         for child in self._children:
-            child.cancel()
-        return True
+            if child.cancel():
+                ret = True
+        return ret
 
 
 def gather(*coros_or_futures, loop=None, return_exceptions=False):
     """Return a future aggregating results from the given coroutines
     or futures.
 
+    Coroutines will be wrapped in a future and scheduled in the event
+    loop. They will not necessarily be scheduled in the same order as
+    passed in.
+
     All futures must share the same event loop.  If all the tasks are
     done successfully, the returned future's result is the list of
     results (in the order of the original sequence, not necessarily
@@ -613,7 +632,7 @@
 
     arg_to_fut = {}
     for arg in set(coros_or_futures):
-        if not isinstance(arg, futures.Future):
+        if not futures.isfuture(arg):
             fut = ensure_future(arg, loop=loop)
             if loop is None:
                 loop = fut._loop
diff --git a/lib-python/3/asyncio/test_utils.py b/lib-python/3/asyncio/test_utils.py
--- a/lib-python/3/asyncio/test_utils.py
+++ b/lib-python/3/asyncio/test_utils.py
@@ -13,6 +13,8 @@
 import threading
 import time
 import unittest
+import weakref
+
 from unittest import mock
 
 from http.server import HTTPServer
@@ -300,6 +302,8 @@
         self.writers = {}
         self.reset_counters()
 
+        self._transports = weakref.WeakValueDictionary()
+
     def time(self):
         return self._time
 
@@ -318,10 +322,10 @@
             else:  # pragma: no cover
                 raise AssertionError("Time generator is not finished")
 
-    def add_reader(self, fd, callback, *args):
+    def _add_reader(self, fd, callback, *args):
         self.readers[fd] = events.Handle(callback, args, self)
 
-    def remove_reader(self, fd):
+    def _remove_reader(self, fd):
         self.remove_reader_count[fd] += 1
         if fd in self.readers:
             del self.readers[fd]
@@ -337,10 +341,10 @@
         assert handle._args == args, '{!r} != {!r}'.format(
             handle._args, args)
 
-    def add_writer(self, fd, callback, *args):
+    def _add_writer(self, fd, callback, *args):
         self.writers[fd] = events.Handle(callback, args, self)
 
-    def remove_writer(self, fd):
+    def _remove_writer(self, fd):
         self.remove_writer_count[fd] += 1
         if fd in self.writers:
             del self.writers[fd]
@@ -356,6 +360,36 @@
         assert handle._args == args, '{!r} != {!r}'.format(
             handle._args, args)
 
+    def _ensure_fd_no_transport(self, fd):
+        try:
+            transport = self._transports[fd]
+        except KeyError:
+            pass
+        else:
+            raise RuntimeError(
+                'File descriptor {!r} is used by transport {!r}'.format(
+                    fd, transport))
+
+    def add_reader(self, fd, callback, *args):
+        """Add a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_reader(fd, callback, *args)
+
+    def remove_reader(self, fd):
+        """Remove a reader callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_reader(fd)
+
+    def add_writer(self, fd, callback, *args):
+        """Add a writer callback.."""
+        self._ensure_fd_no_transport(fd)
+        return self._add_writer(fd, callback, *args)
+
+    def remove_writer(self, fd):
+        """Remove a writer callback."""
+        self._ensure_fd_no_transport(fd)
+        return self._remove_writer(fd)
+
     def reset_counters(self):
         self.remove_reader_count = collections.defaultdict(int)
         self.remove_writer_count = collections.defaultdict(int)
@@ -415,7 +449,13 @@
         self.set_event_loop(loop)
         return loop
 
+    def setUp(self):
+        self._get_running_loop = events._get_running_loop
+        events._get_running_loop = lambda: None
+
     def tearDown(self):
+        events._get_running_loop = self._get_running_loop
+
         events.set_event_loop(None)
 
         # Detect CPython bug #23353: ensure that yield/yield-from is not used
diff --git a/lib-python/3/asyncio/transports.py b/lib-python/3/asyncio/transports.py
--- a/lib-python/3/asyncio/transports.py
+++ b/lib-python/3/asyncio/transports.py
@@ -33,6 +33,14 @@
         """
         raise NotImplementedError
 
+    def set_protocol(self, protocol):
+        """Set a new protocol."""
+        raise NotImplementedError
+
+    def get_protocol(self):
+        """Return the current protocol."""
+        raise NotImplementedError
+
 
 class ReadTransport(BaseTransport):
     """Interface for read-only transports."""
diff --git a/lib-python/3/asyncio/unix_events.py b/lib-python/3/asyncio/unix_events.py
--- a/lib-python/3/asyncio/unix_events.py
+++ b/lib-python/3/asyncio/unix_events.py
@@ -39,6 +39,13 @@
     pass
 
 
+try:
+    _fspath = os.fspath
+except AttributeError:
+    # Python 3.5 or earlier
+    _fspath = lambda path: path
+
+
 class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop):
     """Unix event loop.
 
@@ -234,6 +241,11 @@
         else:
             if sock is None:
                 raise ValueError('no path and sock were specified')
+            if (sock.family != socket.AF_UNIX or
+                    not base_events._is_stream_socket(sock)):
+                raise ValueError(
+                    'A UNIX Domain Stream Socket was expected, got {!r}'
+                    .format(sock))
             sock.setblocking(False)
 
         transport, protocol = yield from self._create_connection_transport(
@@ -251,8 +263,20 @@
                 raise ValueError(
                     'path and sock can not be specified at the same time')
 
+            path = _fspath(path)
             sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 
+            # Check for abstract socket. `str` and `bytes` paths are supported.
+            if path[0] not in (0, '\x00'):
+                try:
+                    if stat.S_ISSOCK(os.stat(path).st_mode):
+                        os.remove(path)
+                except FileNotFoundError:
+                    pass
+                except OSError as err:
+                    # Directory may have permissions only to create socket.
+                    logger.error('Unable to check or remove stale UNIX socket %r: %r', path, err)
+
             try:
                 sock.bind(path)
             except OSError as exc:
@@ -272,9 +296,11 @@
                 raise ValueError(
                     'path was not specified, and no sock specified')
 
-            if sock.family != socket.AF_UNIX:
+            if (sock.family != socket.AF_UNIX or
+                    not base_events._is_stream_socket(sock)):
                 raise ValueError(
-                    'A UNIX Domain Socket was expected, got {!r}'.format(sock))
+                    'A UNIX Domain Stream Socket was expected, got {!r}'
+                    .format(sock))
 
         server = base_events.Server(self, [sock])
         sock.listen(backlog)
@@ -305,17 +331,23 @@
         self._loop = loop
         self._pipe = pipe
         self._fileno = pipe.fileno()
+        self._protocol = protocol
+        self._closing = False
+
         mode = os.fstat(self._fileno).st_mode
         if not (stat.S_ISFIFO(mode) or
                 stat.S_ISSOCK(mode) or
                 stat.S_ISCHR(mode)):
+            self._pipe = None
+            self._fileno = None
+            self._protocol = None
             raise ValueError("Pipe transport is for pipes/sockets only.")
+
         _set_nonblocking(self._fileno)
-        self._protocol = protocol
-        self._closing = False
+
         self._loop.call_soon(self._protocol.connection_made, self)
         # only start reading when connection_made() has been called
-        self._loop.call_soon(self._loop.add_reader,
+        self._loop.call_soon(self._loop._add_reader,
                              self._fileno, self._read_ready)
         if waiter is not None:
             # only wake up the waiter when connection_made() has been called

From pypy.commits at gmail.com  Sun Feb  5 16:46:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Feb 2017 13:46:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge
Message-ID: <58979d2e.cd2f1c0a.44bd1.64d6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89963:748aa3022295
Date: 2017-02-05 22:45 +0100
http://bitbucket.org/pypy/pypy/changeset/748aa3022295/

Log:	hg merge

diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -31,6 +31,26 @@
            "WeakKeyDictionary", "ReferenceError", "ReferenceType", "ProxyType",
            "CallableProxyType", "ProxyTypes", "WeakValueDictionary", 'WeakSet']
 
+try:
+    from __pypy__ import delitem_if_value_is as _delitem_if_value_is
+except ImportError:
+    def _delitem_if_value_is(d, key, value):
+        try:
+            if self.data[key] is value:  # fall-back: there is a potential
+                #             race condition in multithreaded programs HERE
+                del self.data[key]
+        except KeyError:
+            pass
+
+def _remove_dead_weakref(d, key):
+    try:
+        wr = d[key]
+    except KeyError:
+        pass
+    else:
+        if wr() is None:
+            _delitem_if_value_is(d, key, wr)
+
 
 class WeakValueDictionary(UserDict.UserDict):
     """Mapping class that references values weakly.
@@ -58,14 +78,9 @@
                 if self._iterating:
                     self._pending_removals.append(wr.key)
                 else:
-                    # Changed this for PyPy: made more resistent.  The
-                    # issue is that in some corner cases, self.data
-                    # might already be changed or removed by the time
-                    # this weakref's callback is called.  If that is
-                    # the case, we don't want to randomly kill an
-                    # unrelated entry.
-                    if self.data.get(wr.key) is wr:
-                        del self.data[wr.key]
+                    # Atomic removal is necessary since this function
+                    # can be called asynchronously by the GC
+                    _delitem_if_value_is(self.data, wr.key, wr)
         self._remove = remove
         # A list of keys to be removed
         self._pending_removals = []
@@ -78,7 +93,8 @@
         # We shouldn't encounter any KeyError, because this method should
         # always be called *before* mutating the dict.
         while l:
-            del d[l.pop()]
+            key = l.pop()
+            _remove_dead_weakref(d, key)
 
     def __getitem__(self, key):
         o = self.data[key]()
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -78,6 +78,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
+        'delitem_if_value_is'       : 'interp_dict.delitem_if_value_is',
         'move_to_end'               : 'interp_dict.move_to_end',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -59,3 +59,16 @@
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_move_to_end(space, w_key, last)
+
+def delitem_if_value_is(space, w_obj, w_key, w_value):
+    """Atomic equivalent to: 'if dict.get(key) is value: del dict[key]'.
+
+    SPECIAL USE CASES ONLY!  Avoid using on dicts which are specialized,
+    e.g. to int or str keys, because it switches to the object strategy.
+    Also, the 'is' operation is really pointer equality, so avoid using
+    it if 'value' is an immutable object like int or str.
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_delitem_if_value_is(space, w_key, w_value)
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -40,6 +40,7 @@
 import pypy.module.cpyext.pyerrors
 import pypy.module.cpyext.typeobject
 import pypy.module.cpyext.object
+import pypy.module.cpyext.buffer
 import pypy.module.cpyext.bytesobject
 import pypy.module.cpyext.bytearrayobject
 import pypy.module.cpyext.tupleobject
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -126,7 +126,7 @@
 Py_TPFLAGS_HEAPTYPE
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
-PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/buffer.py
@@ -0,0 +1,72 @@
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.interpreter.error import oefmt
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP,
+    generic_cpy_call,
+    PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES, PyBUF_SIMPLE)
+from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
+
+ at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
+def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
+    """Returns a pointer to a read-only memory location usable as
+    character-based input.  The obj argument must support the single-segment
+    character buffer interface.  On success, returns 0, sets buffer to the
+    memory location and size to the buffer length.  Returns -1 and sets a
+    TypeError on error.
+    """
+    pto = obj.c_ob_type
+    pb = pto.c_tp_as_buffer
+    if not (pb and pb.c_bf_getbuffer):
+        raise oefmt(space.w_TypeError,
+                    "expected an object with the buffer interface")
+    with lltype.scoped_alloc(Py_buffer) as view:
+        ret = generic_cpy_call(
+            space, pb.c_bf_getbuffer,
+            obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
+        if rffi.cast(lltype.Signed, ret) == -1:
+            return -1
+
+        bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
+        sizep[0] = view.c_len
+
+        if pb.c_bf_releasebuffer:
+            generic_cpy_call(space, pb.c_bf_releasebuffer,
+                             obj, view)
+        Py_DecRef(space, view.c_obj)
+    return 0
+
+
+ at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
+              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
+def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
+    """
+    Fills in a buffer-info structure correctly for an exporter that can only
+    share a contiguous chunk of memory of "unsigned bytes" of the given
+    length. Returns 0 on success and -1 (with raising an error) on error.
+    """
+    flags = rffi.cast(lltype.Signed, flags)
+    if flags & PyBUF_WRITABLE and readonly:
+        raise oefmt(space.w_ValueError, "Object is not writable")
+    view.c_buf = buf
+    view.c_len = length
+    view.c_obj = obj
+    if obj:
+        Py_IncRef(space, obj)
+    view.c_itemsize = 1
+    rffi.setintfield(view, 'c_readonly', readonly)
+    rffi.setintfield(view, 'c_ndim', 1)
+    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
+    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
+    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
+    return 0
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,13 +1,11 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function, cts,
-    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
+    cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t,
+    PyVarObject, size_t, slot_function, cts,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
-    Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
+    Py_GE, CONST_STRING, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
-    PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
-    get_typedescr, _Py_NewReference)
+    PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef, get_typedescr)
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
@@ -16,10 +14,6 @@
 import pypy.module.__builtin__.operation as operation
 
 
-# from include/object.h
-PyBUF_SIMPLE = 0x0000
-PyBUF_WRITABLE = 0x0001
-
 @cpython_api([size_t], rffi.VOIDP)
 def PyObject_Malloc(space, size):
     # returns non-zero-initialized memory, like CPython
@@ -444,36 +438,6 @@
     is active then NULL is returned but PyErr_Occurred() will return false."""
     return space.call_function(space.builtin.get('dir'), w_o)
 
- at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
-def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
-    """Returns a pointer to a read-only memory location usable as
-    character-based input.  The obj argument must support the single-segment
-    character buffer interface.  On success, returns 0, sets buffer to the
-    memory location and size to the buffer length.  Returns -1 and sets a
-    TypeError on error.
-    """
-    pto = obj.c_ob_type
-
-    pb = pto.c_tp_as_buffer
-    if not (pb and pb.c_bf_getbuffer):
-        raise oefmt(space.w_TypeError,
-                    "expected an object with the buffer interface")
-    with lltype.scoped_alloc(Py_buffer) as view:
-        ret = generic_cpy_call(
-            space, pb.c_bf_getbuffer,
-            obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
-        if rffi.cast(lltype.Signed, ret) == -1:
-            return -1
-
-        bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
-        sizep[0] = view.c_len
-
-        if pb.c_bf_releasebuffer:
-            generic_cpy_call(space, pb.c_bf_releasebuffer,
-                             obj, view)
-        Py_DecRef(space, view.c_obj)
-    return 0
-
 # Also in include/object.h
 Py_PRINT_RAW = 1 # No string quotes etc.
 
@@ -493,41 +457,3 @@
     with rffi.scoped_nonmovingbuffer(data) as buf:
         fwrite(buf, 1, count, fp)
     return 0
-
-
-PyBUF_WRITABLE = 0x0001  # Copied from object.h
-
- at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
-              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
-def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
-    """
-    Fills in a buffer-info structure correctly for an exporter that can only
-    share a contiguous chunk of memory of "unsigned bytes" of the given
-    length. Returns 0 on success and -1 (with raising an error) on error.
-    """
-    flags = rffi.cast(lltype.Signed, flags)
-    if flags & PyBUF_WRITABLE and readonly:
-        raise oefmt(space.w_ValueError, "Object is not writable")
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    if obj:
-        Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 1)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
-        view.c_format = rffi.str2charp("B")
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_ND) == PyBUF_ND:
-        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-        view.c_shape[0] = view.c_len
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
-        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-        view.c_strides[0] = view.c_itemsize
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
-    return 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -548,14 +548,14 @@
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bytes_getbuffer(space, w_str, view, flags):
     from pypy.module.cpyext.bytesobject import PyBytes_AsString
-    from pypy.module.cpyext.object import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
     c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str))
     return PyBuffer_FillInfo(space, view, w_str, c_buf,
                              space.len_w(w_str), 1, flags)
 
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bf_getbuffer(space, w_obj, view, flags):
-    from pypy.module.cpyext.object import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
     buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
     c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     return PyBuffer_FillInfo(space, view, w_obj, c_buf,
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -263,6 +263,14 @@
                     for i in range(len(keys_w)):
                         self.setitem(keys_w[i], values_w[i])
 
+    def nondescr_delitem_if_value_is(self, space, w_key, w_value):
+        """Not exposed directly to app-level, but used by
+        _weakref._remove_dead_weakref and via __pypy__.delitem_if_value_is().
+        """
+        strategy = self.ensure_object_strategy()
+        d = strategy.unerase(self.dstorage)
+        objectmodel.delitem_if_value_is(d, w_key, w_value)
+
     def descr_clear(self, space):
         """D.clear() -> None.  Remove all items from D."""
         self.clear()
@@ -314,11 +322,12 @@
         F: D[k] = F[k]"""
         init_or_update(space, self, __args__, 'dict.update')
 
-    def ensure_object_strategy(self):    # for cpyext
+    def ensure_object_strategy(self):    # also called by cpyext
         object_strategy = self.space.fromcache(ObjectDictStrategy)
         strategy = self.get_strategy()
         if strategy is not object_strategy:
             strategy.switch_to_object_strategy(self)
+        return object_strategy
 
 
 class W_DictObject(W_DictMultiObject):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -293,6 +293,20 @@
                 else:
                     assert list(d) == [key] + other_keys
 
+    def test_delitem_if_value_is(self):
+        import __pypy__
+        class X:
+            pass
+        x2 = X()
+        x3 = X()
+        d = {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x2)
+        assert d == {3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {3: x3}
+
     def test_keys(self):
         d = {1: 2, 3: 4}
         kys = list(d.keys())
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -575,6 +575,10 @@
         pair(self, s_key).delitem()
     method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
 
+    def method_delitem_if_value_is(self, s_key, s_value):
+        pair(self, s_key).setitem(s_value)
+        pair(self, s_key).delitem()
+
 class __extend__(SomeOrderedDict):
 
     def method_move_to_end(self, s_key, s_last):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,20 @@
         return
     d.delitem_with_hash(key, h)
 
+ at specialize.call_location()
+def delitem_if_value_is(d, key, value):
+    """Same as 'if d.get(key) is value: del d[key]'.  It is safe even in
+    case 'd' is an r_dict and the lookup involves callbacks that might
+    release the GIL."""
+    if not we_are_translated():
+        try:
+            if d[key] is value:
+                del d[key]
+        except KeyError:
+            pass
+        return
+    d.delitem_if_value_is(key, value)
+
 def _untranslated_move_to_end(d, key, last):
     "NOT_RPYTHON"
     value = d.pop(key)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config, try_inline, move_to_end)
+    fetch_translated_config, try_inline, delitem_if_value_is, move_to_end)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -661,6 +661,24 @@
     f(29)
     interpret(f, [27])
 
+def test_delitem_if_value_is():
+    class X:
+        pass
+    def f(i):
+        x42 = X()
+        x612 = X()
+        d = {i + .5: x42, i + .6: x612}
+        delitem_if_value_is(d, i + .5, x612)
+        assert (i + .5) in d
+        delitem_if_value_is(d, i + .5, x42)
+        assert (i + .5) not in d
+        delitem_if_value_is(d, i + .5, x612)
+        assert (i + .5) not in d
+        return 0
+
+    f(29)
+    interpret(f, [27])
+
 def test_rdict_with_hash():
     def f(i):
         d = r_dict(strange_key_eq, strange_key_hash)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,12 @@
         hop.exception_is_here()
         hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
 
+    def rtype_method_delitem_if_value_is(self, hop):
+        v_dict, v_key, v_value = hop.inputargs(
+            self, self.key_repr, self.value_repr)
+        hop.exception_cannot_occur()
+        hop.gendirectcall(ll_dict_delitem_if_value_is, v_dict, v_key, v_value)
+
     def rtype_method_move_to_end(self, hop):
         v_dict, v_key, v_last = hop.inputargs(
             self, self.key_repr, lltype.Bool)
@@ -821,6 +827,15 @@
         raise KeyError
     _ll_dict_del(d, hash, index)
 
+def ll_dict_delitem_if_value_is(d, key, value):
+    hash = d.keyhash(key)
+    index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if index < 0:
+        return
+    if d.entries[index].value != value:
+        return
+    _ll_dict_del(d, hash, index)
+
 def _ll_dict_del_entry(d, index):
     d.entries.mark_deleted(index)
     d.num_live_items -= 1

From pypy.commits at gmail.com  Sun Feb  5 18:15:44 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Sun, 05 Feb 2017 15:15:44 -0800 (PST)
Subject: [pypy-commit] cffi sirtom67/float_complex: float complex progress.
 Added __complex__ method to cdata.
Message-ID: <5897b220.5190df0a.5d65e.c657@mx.google.com>

Author: Tom Krauss <thomas.p.krauss at gmail.com>
Branch: sirtom67/float_complex
Changeset: r2879:6dcf51c6003c
Date: 2017-02-05 17:13 -0600
http://bitbucket.org/cffi/cffi/changeset/6dcf51c6003c/

Log:	float complex progress. Added __complex__ method to cdata. Test
	currently failing at > assert repr(complex(cast(p, -0j))) == '-0j' E
	assert '0j' == '-0j' E - 0j E + -0j E ? +

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -884,6 +884,26 @@
     return 0;
 }
 
+static Py_complex
+read_raw_complex_data(char *target, int size)
+{
+    Py_complex r = {.real=0, .imag=0};
+    if (size == 2*sizeof(float)) {
+        float real_part, imag_part;
+        memcpy(&real_part, target + 0,             sizeof(float));
+        memcpy(&imag_part, target + sizeof(float), sizeof(float));
+        r.real = real_part;
+        r.imag = imag_part;
+        return r;
+    }
+    if (size == 2*sizeof(double)) {
+        memcpy(&(r.real), target, 2*sizeof(double));
+        return r;
+    }
+    Py_FatalError("read_raw_complex_data: bad float size");
+    return r;
+}
+
 static void
 write_raw_float_data(char *target, double source, int size)
 {
@@ -1011,6 +1031,10 @@
             return (PyObject *)cd;
         }
     }
+    else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        Py_complex value = read_raw_complex_data(data, ct->ct_size);
+        return PyComplex_FromCComplex(value);
+    }
     else if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
         /*READ(data, ct->ct_size)*/
         if (ct->ct_size == sizeof(char))
@@ -2009,6 +2033,10 @@
         }
         return PyFloat_FromDouble(value);
     }
+    if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        double value = read_raw_float_data(cd->c_data, cd->c_type->ct_size);
+        return PyComplex_FromDoubles(value, 0.0);
+    }
     PyErr_Format(PyExc_TypeError, "float() not supported on cdata '%s'",
                  cd->c_type->ct_name);
     return NULL;
@@ -2808,6 +2836,19 @@
     }
 }
 
+static PyObject *cdata_complex(PyObject *cd_, PyObject *noarg)
+{
+    CDataObject *cd = (CDataObject *)cd_;
+
+    if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        Py_complex value = read_raw_complex_data(cd->c_data, cd->c_type->ct_size);
+        return PyComplex_FromCComplex(value);
+    }
+    PyErr_Format(PyExc_TypeError, "complex() not supported on cdata '%s'",
+                 cd->c_type->ct_name);
+    return NULL;
+}
+
 static PyObject *cdata_iter(CDataObject *);
 
 static PyNumberMethods CData_as_number = {
@@ -2857,8 +2898,9 @@
 };
 
 static PyMethodDef cdata_methods[] = {
-    {"__dir__",   cdata_dir,      METH_NOARGS},
-    {NULL,        NULL}           /* sentinel */
+    {"__dir__",     cdata_dir,      METH_NOARGS},
+    {"__complex__", cdata_complex,  METH_NOARGS},
+    {NULL,          NULL}           /* sentinel */
 };
 
 static PyTypeObject CData_Type = {
@@ -3599,7 +3641,6 @@
         /* cast to a complex */
         Py_complex value;
         PyObject *io;
-        printf("_cffi_backend.c do_cast  ct->ct_size=%ld\n", ct->ct_size);
         if (CData_Check(ob)) {
             CDataObject *cdsrc = (CDataObject *)ob;
 
@@ -3983,8 +4024,6 @@
     int name_size;
     ffi_type *ffitype;
 
-    printf("hello\n");
-
     for (ptypes=types; ; ptypes++) {
         if (ptypes->name == NULL) {
 #ifndef HAVE_WCHAR_H
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -101,7 +101,6 @@
 
 static PyObject *build_primitive_type(int num)
 {
-    fprintf(stderr, "fooooooooooooo    num=%d\n",num);
     /* XXX too many translations between here and new_primitive_type() */
     static const char *primitive_name[] = {
         NULL,
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -197,12 +197,12 @@
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert float(cast(p, INF*1j)) == INF*1j
-        assert float(cast(p, -INF)) == -INF
+        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
+        assert complex(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
+            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
 
         assert cast(p, -1.1j) != cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'

From pypy.commits at gmail.com  Mon Feb  6 03:01:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:01:38 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test: now only three residual calls
 instead of four
Message-ID: <58982d62.d4b81c0a.ac655.9d66@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89964:bff2e6940cc6
Date: 2017-02-06 09:00 +0100
http://bitbucket.org/pypy/pypy/changeset/bff2e6940cc6/

Log:	fix test: now only three residual calls instead of four

diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -364,7 +364,7 @@
                     if n in mdict:
                         raise Exception
         self.meta_interp(f, [10])
-        self.check_simple_loop(call_may_force_i=0, call_i=3, call_n=1)
+        self.check_simple_loop(call_may_force_i=0, call_i=2, call_n=1)
 
     def test_dict_virtual(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto')

From pypy.commits at gmail.com  Mon Feb  6 03:09:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:09:11 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <58982f27.6787df0a.b4168.3c5d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89965:aa5558245c78
Date: 2017-02-06 09:08 +0100
http://bitbucket.org/pypy/pypy/changeset/aa5558245c78/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -134,3 +134,11 @@
 crypographically good way).  It is the default in PyPy3.  The default of
 PyPy2 remains unchanged: there are user programs out there that depend
 on constant hashes (or even sometimes on specific hash results).
+
+.. branch: dict-move-to-end
+
+Our dicts, which are always ordered, now have an extra "method" for
+Python 3.x which moves an item to first or last position.  In PyPy 3.5
+it is the standard ``OrderedDict.move_to_end()`` method, but the
+behavior is also available on Python 2.x or for the ``dict`` type by
+calling ``__pypy__.move_to_end(dict, key, last=True)``.

From pypy.commits at gmail.com  Mon Feb  6 03:13:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:13:44 -0800 (PST)
Subject: [pypy-commit] cffi default: fix the FakeBackend classes for tests
Message-ID: <58983038.0a081c0a.d0dc.9b97@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2880:be4ee14ac677
Date: 2017-02-06 09:10 +0100
http://bitbucket.org/cffi/cffi/changeset/be4ee14ac677/

Log:	fix the FakeBackend classes for tests

diff --git a/testing/cffi0/test_cdata.py b/testing/cffi0/test_cdata.py
--- a/testing/cffi0/test_cdata.py
+++ b/testing/cffi0/test_cdata.py
@@ -26,6 +26,8 @@
     def _get_types(self):
         return "CData", "CType"
 
+    buffer = "buffer type"
+
 
 class FakeType(object):
     def __init__(self, cdecl):
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -47,6 +47,8 @@
     def _get_types(self):
         return "CData", "CType"
 
+    buffer = "buffer type"
+
 class FakeType(object):
     def __init__(self, cdecl):
         self.cdecl = cdecl

From pypy.commits at gmail.com  Mon Feb  6 03:14:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:14:13 -0800 (PST)
Subject: [pypy-commit] pypy default: import cffi/be4ee14ac677
Message-ID: <58983055.b6a9df0a.3bf85.593d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89966:ed21fb40c33b
Date: 2017-02-06 09:11 +0100
http://bitbucket.org/pypy/pypy/changeset/ed21fb40c33b/

Log:	import cffi/be4ee14ac677

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py
@@ -27,6 +27,8 @@
     def _get_types(self):
         return "CData", "CType"
 
+    buffer = "buffer type"
+
 
 class FakeType(object):
     def __init__(self, cdecl):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -48,6 +48,8 @@
     def _get_types(self):
         return "CData", "CType"
 
+    buffer = "buffer type"
+
 class FakeType(object):
     def __init__(self, cdecl):
         self.cdecl = cdecl

From pypy.commits at gmail.com  Mon Feb  6 03:17:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:17:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix?
Message-ID: <58983131.d699df0a.3867b.00bf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89967:2a99fc703ecc
Date: 2017-02-06 09:17 +0100
http://bitbucket.org/pypy/pypy/changeset/2a99fc703ecc/

Log:	fix?

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -71,6 +71,7 @@
     PROTOCOL_SSLv2  = 0
 PROTOCOL_SSLv3  = 1
 PROTOCOL_SSLv23 = 2
+PROTOCOL_TLS    = PROTOCOL_SSLv23
 PROTOCOL_TLSv1    = 3
 if lib.Cryptography_HAS_TLSv1_2:
     PROTOCOL_TLSv1 = 3

From pypy.commits at gmail.com  Mon Feb  6 03:24:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:24:41 -0800 (PST)
Subject: [pypy-commit] cffi default: Merged in
 coronafire/cffi/buffer_richcompare (pull request #76)
Message-ID: <589832c9.63afdf0a.bc7c.0176@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r2884:4abb5f4db011
Date: 2017-02-06 08:24 +0000
http://bitbucket.org/cffi/cffi/changeset/4abb5f4db011/

Log:	Merged in coronafire/cffi/buffer_richcompare (pull request #76)

	Add richcompare to buffer objects

diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -155,6 +155,87 @@
     return 0;
 }
 
+static PyObject *
+mb_richcompare(PyObject *self, PyObject *other, int op)
+{
+    Py_ssize_t self_size, other_size;
+    Py_buffer self_bytes, other_bytes;
+    PyObject *res;
+    Py_ssize_t minsize;
+    int cmp, rc;
+
+    /* Bytes can be compared to anything that supports the (binary)
+       buffer API.  Except that a comparison with Unicode is always an
+       error, even if the comparison is for equality. */
+    rc = PyObject_IsInstance(self, (PyObject*)&PyUnicode_Type);
+    if (!rc)
+        rc = PyObject_IsInstance(other, (PyObject*)&PyUnicode_Type);
+    if (rc < 0)
+        return NULL;
+    if (rc) {
+        if (Py_BytesWarningFlag && (op == Py_EQ || op == Py_NE)) {
+            if (PyErr_WarnEx(PyExc_BytesWarning,
+                            "Comparison between bytearray and string", 1))
+                return NULL;
+        }
+
+        Py_INCREF(Py_NotImplemented);
+        return Py_NotImplemented;
+    }
+
+    if (PyObject_GetBuffer(self, &self_bytes, PyBUF_SIMPLE) != 0) {
+        PyErr_Clear();
+        Py_INCREF(Py_NotImplemented);
+        return Py_NotImplemented;
+
+    }
+    self_size = self_bytes.len;
+
+    if (PyObject_GetBuffer(other, &other_bytes, PyBUF_SIMPLE) != 0) {
+        PyErr_Clear();
+        PyBuffer_Release(&self_bytes);
+        Py_INCREF(Py_NotImplemented);
+        return Py_NotImplemented;
+
+    }
+    other_size = other_bytes.len;
+
+    if (self_size != other_size && (op == Py_EQ || op == Py_NE)) {
+        /* Shortcut: if the lengths differ, the objects differ */
+        cmp = (op == Py_NE);
+    }
+    else {
+        minsize = self_size;
+        if (other_size < minsize)
+            minsize = other_size;
+
+        cmp = memcmp(self_bytes.buf, other_bytes.buf, minsize);
+        /* In ISO C, memcmp() guarantees to use unsigned bytes! */
+
+        if (cmp == 0) {
+            if (self_size < other_size)
+                cmp = -1;
+            else if (self_size > other_size)
+                cmp = 1;
+        }
+
+        switch (op) {
+        case Py_LT: cmp = cmp <  0; break;
+        case Py_LE: cmp = cmp <= 0; break;
+        case Py_EQ: cmp = cmp == 0; break;
+        case Py_NE: cmp = cmp != 0; break;
+        case Py_GT: cmp = cmp >  0; break;
+        case Py_GE: cmp = cmp >= 0; break;
+        }
+    }
+
+    res = cmp ? Py_True : Py_False;
+    PyBuffer_Release(&self_bytes);
+    PyBuffer_Release(&other_bytes);
+    Py_INCREF(res);
+    return res;
+}
+
 #if PY_MAJOR_VERSION >= 3
 /* pfffffffffffff pages of copy-paste from listobject.c */
 static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item)
@@ -287,7 +368,7 @@
     ffi_buffer_doc,                             /* tp_doc */
     (traverseproc)mb_traverse,                  /* tp_traverse */
     (inquiry)mb_clear,                          /* tp_clear */
-    0,                                          /* tp_richcompare */
+    (richcmpfunc)mb_richcompare,                /* tp_richcompare */
     offsetof(MiniBufferObj, mb_weakreflist),    /* tp_weaklistoffset */
     0,                                          /* tp_iter */
     0,                                          /* tp_iternext */
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1226,6 +1226,23 @@
         assert list(a)[:1000] + [0] * (len(a)-1000) == list(b)
         f.close()
 
+    def test_ffi_buffer_comparisons(self):
+        ffi = FFI(backend=self.Backend())
+        ba = bytearray(range(100, 110))
+        a = ffi.new("uint8_t[]", list(ba))
+        try:
+            b_full = ffi.buffer(a)
+            b_short = ffi.buffer(a, 3)
+            b_mid = ffi.buffer(a, 6)
+        except NotImplementedError as e:
+            py.test.skip(str(e))
+        else:
+            content = b_full[:]
+            assert content == b_full == ba
+            assert b_short < b_mid < b_full
+            assert ba > b_mid > ba[0:2]
+            assert b_short != ba[1:4]
+
     def test_array_in_struct(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct foo_s { int len; short data[5]; };")
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -191,10 +191,10 @@
         s = ffi.new("struct s1 *")
         setattr(s, name, value)
         assert getattr(s, name) == value
-        raw1 = ffi.buffer(s)[:]
+        buff1 = ffi.buffer(s)
         t = lib.try_with_value(fnames.index(name), value)
-        raw2 = ffi.buffer(t, len(raw1))[:]
-        assert raw1 == raw2
+        buff2 = ffi.buffer(t, len(buff1))
+        assert buff1 == buff2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)

From pypy.commits at gmail.com  Mon Feb  6 03:24:48 2017
From: pypy.commits at gmail.com (coronafire)
Date: Mon, 06 Feb 2017 00:24:48 -0800 (PST)
Subject: [pypy-commit] cffi buffer_richcompare: Add tests for buffer
 comparisons
Message-ID: <589832d0.cd2f1c0a.44bd1.0263@mx.google.com>

Author: Andrew Leech <andrew at alelec.net>
Branch: buffer_richcompare
Changeset: r2882:84ec6d83b6fe
Date: 2017-02-03 13:03 +1100
http://bitbucket.org/cffi/cffi/changeset/84ec6d83b6fe/

Log:	Add tests for buffer comparisons

diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1225,6 +1225,23 @@
         assert list(a)[:1000] + [0] * (len(a)-1000) == list(b)
         f.close()
 
+    def test_ffi_buffer_comparisons(self):
+        ffi = FFI(backend=self.Backend())
+        ba = bytearray(range(100, 110))
+        a = ffi.new("uint8_t[]", list(ba))
+        try:
+            b_full = ffi.buffer(a)
+            b_short = ffi.buffer(a, 3)
+            b_mid = ffi.buffer(a, 6)
+        except NotImplementedError as e:
+            py.test.skip(str(e))
+        else:
+            content = b_full[:]
+            assert content == b_full == ba
+            assert b_short < b_mid < b_full
+            assert ba > b_mid > ba[0:2]
+            assert b_short != ba[1:4]
+
     def test_array_in_struct(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct foo_s { int len; short data[5]; };")
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -191,10 +191,10 @@
         s = ffi.new("struct s1 *")
         setattr(s, name, value)
         assert getattr(s, name) == value
-        raw1 = ffi.buffer(s)[:]
+        buff1 = ffi.buffer(s)
         t = lib.try_with_value(fnames.index(name), value)
-        raw2 = ffi.buffer(t, len(raw1))[:]
-        assert raw1 == raw2
+        buff2 = ffi.buffer(t, len(buff1))
+        assert buff1 == buff2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)

From pypy.commits at gmail.com  Mon Feb  6 03:24:46 2017
From: pypy.commits at gmail.com (coronafire)
Date: Mon, 06 Feb 2017 00:24:46 -0800 (PST)
Subject: [pypy-commit] cffi buffer_richcompare: Add tp_richcompare from 3.5
 bytearray to minibuffer (ffi.buffer)
Message-ID: <589832ce.49acdf0a.9bc36.4b3d@mx.google.com>

Author: Andrew Leech <andrew at alelec.net>
Branch: buffer_richcompare
Changeset: r2881:d52ba2920e1c
Date: 2017-02-03 13:01 +1100
http://bitbucket.org/cffi/cffi/changeset/d52ba2920e1c/

Log:	Add tp_richcompare from 3.5 bytearray to minibuffer (ffi.buffer)

diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -155,6 +155,87 @@
     return 0;
 }
 
+static PyObject *
+mb_richcompare(PyObject *self, PyObject *other, int op)
+{
+    Py_ssize_t self_size, other_size;
+    Py_buffer self_bytes, other_bytes;
+    PyObject *res;
+    Py_ssize_t minsize;
+    int cmp, rc;
+
+    /* Bytes can be compared to anything that supports the (binary)
+       buffer API.  Except that a comparison with Unicode is always an
+       error, even if the comparison is for equality. */
+    rc = PyObject_IsInstance(self, (PyObject*)&PyUnicode_Type);
+    if (!rc)
+        rc = PyObject_IsInstance(other, (PyObject*)&PyUnicode_Type);
+    if (rc < 0)
+        return NULL;
+    if (rc) {
+        if (Py_BytesWarningFlag && (op == Py_EQ || op == Py_NE)) {
+            if (PyErr_WarnEx(PyExc_BytesWarning,
+                            "Comparison between bytearray and string", 1))
+                return NULL;
+        }
+
+        Py_INCREF(Py_NotImplemented);
+        return Py_NotImplemented;
+    }
+
+    if (PyObject_GetBuffer(self, &self_bytes, PyBUF_SIMPLE) != 0) {
+        PyErr_Clear();
+        Py_INCREF(Py_NotImplemented);
+        return Py_NotImplemented;
+
+    }
+    self_size = self_bytes.len;
+
+    if (PyObject_GetBuffer(other, &other_bytes, PyBUF_SIMPLE) != 0) {
+        PyErr_Clear();
+        PyBuffer_Release(&self_bytes);
+        Py_INCREF(Py_NotImplemented);
+        return Py_NotImplemented;
+
+    }
+    other_size = other_bytes.len;
+
+    if (self_size != other_size && (op == Py_EQ || op == Py_NE)) {
+        /* Shortcut: if the lengths differ, the objects differ */
+        cmp = (op == Py_NE);
+    }
+    else {
+        minsize = self_size;
+        if (other_size < minsize)
+            minsize = other_size;
+
+        cmp = memcmp(self_bytes.buf, other_bytes.buf, minsize);
+        /* In ISO C, memcmp() guarantees to use unsigned bytes! */
+
+        if (cmp == 0) {
+            if (self_size < other_size)
+                cmp = -1;
+            else if (self_size > other_size)
+                cmp = 1;
+        }
+
+        switch (op) {
+        case Py_LT: cmp = cmp <  0; break;
+        case Py_LE: cmp = cmp <= 0; break;
+        case Py_EQ: cmp = cmp == 0; break;
+        case Py_NE: cmp = cmp != 0; break;
+        case Py_GT: cmp = cmp >  0; break;
+        case Py_GE: cmp = cmp >= 0; break;
+        }
+    }
+
+    res = cmp ? Py_True : Py_False;
+    PyBuffer_Release(&self_bytes);
+    PyBuffer_Release(&other_bytes);
+    Py_INCREF(res);
+    return res;
+}
+
 #if PY_MAJOR_VERSION >= 3
 /* pfffffffffffff pages of copy-paste from listobject.c */
 static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item)

From pypy.commits at gmail.com  Mon Feb  6 03:24:50 2017
From: pypy.commits at gmail.com (coronafire)
Date: Mon, 06 Feb 2017 00:24:50 -0800 (PST)
Subject: [pypy-commit] cffi buffer_richcompare: Enable the richcompare
 function in minibuffer
Message-ID: <589832d2.85b2df0a.47768.053b@mx.google.com>

Author: Andrew Leech <andrew at alelec.net>
Branch: buffer_richcompare
Changeset: r2883:262d319d4f4b
Date: 2017-02-05 12:59 +1100
http://bitbucket.org/cffi/cffi/changeset/262d319d4f4b/

Log:	Enable the richcompare function in minibuffer

diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -352,7 +352,7 @@
     0,                                          /* tp_doc */
     (traverseproc)mb_traverse,                  /* tp_traverse */
     (inquiry)mb_clear,                          /* tp_clear */
-    0,                                          /* tp_richcompare */
+    (richcmpfunc)mb_richcompare,                /* tp_richcompare */
     offsetof(MiniBufferObj, mb_weakreflist),    /* tp_weaklistoffset */
 };
 

From pypy.commits at gmail.com  Mon Feb  6 03:34:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:34:09 -0800 (PST)
Subject: [pypy-commit] cffi default: Remove a warning that doesn't really
 make sense here, tweak tests
Message-ID: <58983501.ce9adf0a.f9907.00cb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2885:e46941e99bf4
Date: 2017-02-06 09:33 +0100
http://bitbucket.org/cffi/cffi/changeset/e46941e99bf4/

Log:	Remove a warning that doesn't really make sense here, tweak tests

diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -173,12 +173,6 @@
     if (rc < 0)
         return NULL;
     if (rc) {
-        if (Py_BytesWarningFlag && (op == Py_EQ || op == Py_NE)) {
-            if (PyErr_WarnEx(PyExc_BytesWarning,
-                            "Comparison between bytearray and string", 1))
-                return NULL;
-        }
-
         Py_INCREF(Py_NotImplemented);
         return Py_NotImplemented;
     }
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1229,17 +1229,20 @@
     def test_ffi_buffer_comparisons(self):
         ffi = FFI(backend=self.Backend())
         ba = bytearray(range(100, 110))
+        assert ba == memoryview(ba)    # justification for the following
         a = ffi.new("uint8_t[]", list(ba))
+        c = ffi.new("uint8_t[]", [99] + list(ba))
         try:
             b_full = ffi.buffer(a)
             b_short = ffi.buffer(a, 3)
             b_mid = ffi.buffer(a, 6)
+            b_other = ffi.buffer(c, 6)
         except NotImplementedError as e:
             py.test.skip(str(e))
         else:
             content = b_full[:]
             assert content == b_full == ba
-            assert b_short < b_mid < b_full
+            assert b_other < b_short < b_mid < b_full
             assert ba > b_mid > ba[0:2]
             assert b_short != ba[1:4]
 
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -191,8 +191,11 @@
         s = ffi.new("struct s1 *")
         setattr(s, name, value)
         assert getattr(s, name) == value
+        raw1 = ffi.buffer(s)[:]
         buff1 = ffi.buffer(s)
         t = lib.try_with_value(fnames.index(name), value)
+        raw2 = ffi.buffer(t, len(raw1))[:]
+        assert raw1 == raw2
         buff2 = ffi.buffer(t, len(buff1))
         assert buff1 == buff2
 

From pypy.commits at gmail.com  Mon Feb  6 03:57:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 00:57:46 -0800 (PST)
Subject: [pypy-commit] pypy default: update to cffi/e46941e99bf4
Message-ID: <58983a8a.484a1c0a.25f33.b498@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89968:3e215d0078ab
Date: 2017-02-06 09:56 +0100
http://bitbucket.org/pypy/pypy/changeset/3e215d0078ab/

Log:	update to cffi/e46941e99bf4

diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -56,6 +56,62 @@
                 e.w_type = space.w_ValueError
             raise
 
+    def _comparison_helper(self, space, w_other, mode):
+        if space.isinstance_w(w_other, space.w_unicode):
+            return space.w_NotImplemented
+        try:
+            other_buf = space.buffer_w(w_other, space.BUF_SIMPLE)
+        except OperationError as e:
+            if e.async(space):
+                raise
+            return space.w_NotImplemented
+        my_buf = self.buf
+        my_len = len(my_buf)
+        other_len = len(other_buf)
+        if other_len != my_len:
+            if mode == 'E':
+                return space.w_False
+            if mode == 'N':
+                return space.w_True
+        cmp = _memcmp(my_buf, other_buf, min(my_len, other_len))
+        if cmp == 0:
+            if my_len < other_len:
+                cmp = -1
+            elif my_len > other_len:
+                cmp = 1
+
+        if   mode == 'L': res = cmp <  0
+        elif mode == 'l': res = cmp <= 0
+        elif mode == 'E': res = cmp == 0
+        elif mode == 'N': res = cmp != 0
+        elif mode == 'G': res = cmp >  0
+        elif mode == 'g': res = cmp >= 0
+        else: raise AssertionError
+
+        return space.newbool(res)
+
+    def descr_eq(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'E')
+    def descr_ne(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'N')
+    def descr_lt(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'L')
+    def descr_le(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'l')
+    def descr_gt(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'G')
+    def descr_ge(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'g')
+
+def _memcmp(buf1, buf2, length):
+    # XXX very slow
+    for i in range(length):
+        if buf1[i] < buf2[i]:
+            return -1
+        if buf1[i] > buf2[i]:
+            return 1
+    return 0
+
 @unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
 def MiniBuffer___new__(space, w_subtype, w_cdata, size=-1):
     ctype = w_cdata.ctype
@@ -85,6 +141,12 @@
     __len__ = interp2app(MiniBuffer.descr_len),
     __getitem__ = interp2app(MiniBuffer.descr_getitem),
     __setitem__ = interp2app(MiniBuffer.descr_setitem),
+    __eq__ = interp2app(MiniBuffer.descr_eq),
+    __ne__ = interp2app(MiniBuffer.descr_ne),
+    __lt__ = interp2app(MiniBuffer.descr_lt),
+    __le__ = interp2app(MiniBuffer.descr_le),
+    __gt__ = interp2app(MiniBuffer.descr_gt),
+    __ge__ = interp2app(MiniBuffer.descr_ge),
     __weakref__ = make_weakref_descr(MiniBuffer),
     __str__ = interp2app(MiniBuffer.descr_str),
     __doc__ = """ffi.buffer(cdata[, byte_size]):
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -259,6 +259,25 @@
         assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
         assert type(ffi.buffer(a)) is ffi.buffer
 
+    def test_ffi_buffer_comparisons(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        ba = bytearray(range(100, 110))
+        assert ba == memoryview(ba)    # justification for the following
+        a = ffi.new("uint8_t[]", list(ba))
+        c = ffi.new("uint8_t[]", [99] + list(ba))
+        b_full = ffi.buffer(a)
+        b_short = ffi.buffer(a, 3)
+        b_mid = ffi.buffer(a, 6)
+        b_other = ffi.buffer(c, 6)
+        content = b_full[:]
+        assert content == b_full == ba
+        assert b_short < b_mid < b_full
+        assert b_other < b_short < b_mid < b_full
+        assert ba > b_mid > ba[0:2]
+        assert b_short != ba[1:4]
+        assert b_short != 42
+
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend
         import array
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1254,6 +1254,7 @@
 
 @specialize.argtype(1)
 def _memcmp(selfvalue, buffer, length):
+    # XXX that's very slow if selfvalue or buffer are Buffer objects
     for i in range(length):
         if selfvalue[i] < buffer[i]:
             return -1

From pypy.commits at gmail.com  Mon Feb  6 04:13:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 01:13:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58983e4c.da6d1c0a.67a1d.c2fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89969:451124f17d87
Date: 2017-02-06 10:13 +0100
http://bitbucket.org/pypy/pypy/changeset/451124f17d87/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -143,3 +143,11 @@
 crypographically good way).  It is the default in PyPy3.  The default of
 PyPy2 remains unchanged: there are user programs out there that depend
 on constant hashes (or even sometimes on specific hash results).
+
+.. branch: dict-move-to-end
+
+Our dicts, which are always ordered, now have an extra "method" for
+Python 3.x which moves an item to first or last position.  In PyPy 3.5
+it is the standard ``OrderedDict.move_to_end()`` method, but the
+behavior is also available on Python 2.x or for the ``dict`` type by
+calling ``__pypy__.move_to_end(dict, key, last=True)``.
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, interp2app
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
@@ -75,6 +75,62 @@
             self.buffer.setslice(start, value.as_str())
 
 
+    def _comparison_helper(self, space, w_other, mode):
+        if space.isinstance_w(w_other, space.w_unicode):
+            return space.w_NotImplemented
+        try:
+            other_buf = space.buffer_w(w_other, space.BUF_SIMPLE)
+        except OperationError as e:
+            if e.async(space):
+                raise
+            return space.w_NotImplemented
+        my_buf = self.buffer
+        my_len = len(my_buf)
+        other_len = len(other_buf)
+        if other_len != my_len:
+            if mode == 'E':
+                return space.w_False
+            if mode == 'N':
+                return space.w_True
+        cmp = _memcmp(my_buf, other_buf, min(my_len, other_len))
+        if cmp == 0:
+            if my_len < other_len:
+                cmp = -1
+            elif my_len > other_len:
+                cmp = 1
+
+        if   mode == 'L': res = cmp <  0
+        elif mode == 'l': res = cmp <= 0
+        elif mode == 'E': res = cmp == 0
+        elif mode == 'N': res = cmp != 0
+        elif mode == 'G': res = cmp >  0
+        elif mode == 'g': res = cmp >= 0
+        else: raise AssertionError
+
+        return space.newbool(res)
+
+    def descr_eq(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'E')
+    def descr_ne(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'N')
+    def descr_lt(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'L')
+    def descr_le(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'l')
+    def descr_gt(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'G')
+    def descr_ge(self, space, w_other):
+        return self._comparison_helper(space, w_other, 'g')
+
+def _memcmp(buf1, buf2, length):
+    # XXX very slow
+    for i in range(length):
+        if buf1[i] < buf2[i]:
+            return -1
+        if buf1[i] > buf2[i]:
+            return 1
+    return 0
+
 @unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
 def MiniBuffer___new__(space, w_subtype, w_cdata, size=-1):
     ctype = w_cdata.ctype
@@ -104,6 +160,12 @@
     __len__ = interp2app(MiniBuffer.descr_len),
     __getitem__ = interp2app(MiniBuffer.descr_getitem),
     __setitem__ = interp2app(MiniBuffer.descr_setitem),
+    __eq__ = interp2app(MiniBuffer.descr_eq),
+    __ne__ = interp2app(MiniBuffer.descr_ne),
+    __lt__ = interp2app(MiniBuffer.descr_lt),
+    __le__ = interp2app(MiniBuffer.descr_le),
+    __gt__ = interp2app(MiniBuffer.descr_gt),
+    __ge__ = interp2app(MiniBuffer.descr_ge),
     __weakref__ = make_weakref_descr(MiniBuffer),
     __doc__ = """ffi.buffer(cdata[, byte_size]):
 Return a read-write buffer object that references the raw C data
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -259,6 +259,25 @@
         assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
         assert type(ffi.buffer(a)) is ffi.buffer
 
+    def test_ffi_buffer_comparisons(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        ba = bytearray(range(100, 110))
+        assert ba == memoryview(ba)    # justification for the following
+        a = ffi.new("uint8_t[]", list(ba))
+        c = ffi.new("uint8_t[]", [99] + list(ba))
+        b_full = ffi.buffer(a)
+        b_short = ffi.buffer(a, 3)
+        b_mid = ffi.buffer(a, 6)
+        b_other = ffi.buffer(c, 6)
+        content = b_full[:]
+        assert content == b_full == ba
+        assert b_short < b_mid < b_full
+        assert b_other < b_short < b_mid < b_full
+        assert ba > b_mid > ba[0:2]
+        assert b_short != ba[1:4]
+        assert b_short != 42
+
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend
         import array
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py
@@ -27,6 +27,8 @@
     def _get_types(self):
         return "CData", "CType"
 
+    buffer = "buffer type"
+
 
 class FakeType(object):
     def __init__(self, cdecl):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -48,6 +48,8 @@
     def _get_types(self):
         return "CData", "CType"
 
+    buffer = "buffer type"
+
 class FakeType(object):
     def __init__(self, cdecl):
         self.cdecl = cdecl
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1324,6 +1324,7 @@
 
 @specialize.argtype(1)
 def _memcmp(selfvalue, buffer, length):
+    # XXX that's very slow if selfvalue or buffer are Buffer objects
     for i in range(length):
         if selfvalue[i] < buffer[i]:
             return -1
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -364,7 +364,7 @@
                     if n in mdict:
                         raise Exception
         self.meta_interp(f, [10])
-        self.check_simple_loop(call_may_force_i=0, call_i=3, call_n=1)
+        self.check_simple_loop(call_may_force_i=0, call_i=2, call_n=1)
 
     def test_dict_virtual(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto')

From pypy.commits at gmail.com  Mon Feb  6 07:05:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 04:05:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix error message
Message-ID: <5898669a.84301c0a.98f85.c30d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89970:f0493a5ae4d6
Date: 2017-02-06 13:05 +0100
http://bitbucket.org/pypy/pypy/changeset/f0493a5ae4d6/

Log:	fix error message

diff --git a/lib-python/3/test/test_extcall.py b/lib-python/3/test/test_extcall.py
--- a/lib-python/3/test/test_extcall.py
+++ b/lib-python/3/test/test_extcall.py
@@ -57,7 +57,7 @@
     Traceback (most recent call last):
         ...
     TypeError: ...got multiple values for keyword argument 'a'
-    >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5})
+    >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5})               #doctest: +ELLIPSIS
     Traceback (most recent call last):
         ...
     TypeError: ...got multiple values for keyword argument 'a'
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1470,8 +1470,13 @@
         for i in range(itemcount-1, -1, -1):
             w_item = self.peekvalue(i)
             if not space.ismapping_w(w_item):
-                raise oefmt(space.w_TypeError,
-                            "'%T' object is not a mapping", w_item)
+                if not with_call:
+                    raise oefmt(space.w_TypeError,
+                                "'%T' object is not a mapping", w_item)
+                else:
+                    raise oefmt(space.w_TypeError,
+                                "argument after ** must be a mapping, not %T",
+                                w_item)
             if with_call:
                 expected_length += space.len_w(w_item)
             space.call_method(w_dict, 'update', w_item)
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -340,7 +340,7 @@
         assert self.codetest(code, "g1", []) == (3, 7)
         resg2 = self.codetest(code, 'g2', [])
         assert "TypeError:" in resg2
-        assert "'list' object is not a mapping" in resg2
+        assert "argument after ** must be a mapping, not list" in resg2
         resg3 = self.codetest(code, 'g3', [])
         assert "TypeError:" in resg3
         assert "keywords must be strings" in resg3

From pypy.commits at gmail.com  Mon Feb  6 07:10:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 04:10:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: try to fix test_functools.py
Message-ID: <589867b8.8dae1c0a.5be76.0bb4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89971:90e9fb539547
Date: 2017-02-06 13:09 +0100
http://bitbucket.org/pypy/pypy/changeset/90e9fb539547/

Log:	try to fix test_functools.py

diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -2,6 +2,8 @@
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
 
+from reprlib import recursive_repr as _recursive_repr
+
 
 sentinel = object()
 
@@ -76,6 +78,7 @@
             fkeywords = dict(self._keywords, **fkeywords)
         return self._func(*(self._args + fargs), **fkeywords)
 
+    @_recursive_repr()
     def __repr__(self):
         cls = type(self)
         if cls is partial:

From pypy.commits at gmail.com  Mon Feb  6 07:48:42 2017
From: pypy.commits at gmail.com (PavloKapyshin)
Date: Mon, 06 Feb 2017 04:48:42 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix typo
Message-ID: <589870aa.50a4df0a.126fc.6fb6@mx.google.com>

Author: PavloKapyshin <i at 93z.org>
Branch: 
Changeset: r89972:3ba228f90388
Date: 2017-02-06 12:27 +0000
http://bitbucket.org/pypy/pypy/changeset/3ba228f90388/

Log:	Fix typo

diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -14,7 +14,7 @@
 
 * Write them in C++ and bind them through  :doc:`cppyy <cppyy>` using Cling.
 
-* Write them in as `RPython mixed modules`_.
+* Write them as `RPython mixed modules`_.
 
 
 CFFI

From pypy.commits at gmail.com  Mon Feb  6 10:15:04 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 06 Feb 2017 07:15:04 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: copied over modified vmprof files
 from vmprof-python.git repo
Message-ID: <589892f8.46bb1c0a.499e5.4f5f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89973:34fc984fe649
Date: 2017-02-06 14:21 +0100
http://bitbucket.org/pypy/pypy/changeset/34fc984fe649/

Log:	copied over modified vmprof files from vmprof-python.git repo

diff too long, truncating to 2000 out of 55586 lines

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -14,16 +14,40 @@
 
 ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof')
 SRC = ROOT.join('src')
+UDIS86 = ROOT.join('libudis86')
+BACKTRACE = ROOT.join('libbacktrace')
 
 if sys.platform.startswith('linux'):
+    extra_compile = [
+       BACKTRACE.join('backtrace.c'),
+       BACKTRACE.join('state.c'),
+       BACKTRACE.join('elf.c'),
+       BACKTRACE.join('dwarf.c'),
+       BACKTRACE.join('fileline.c'),
+       BACKTRACE.join('mmap.c'),
+       BACKTRACE.join('mmapio.c'),
+       BACKTRACE.join('posix.c'),
+       BACKTRACE.join('sort.c'),
+    ]
     _libs = ['dl']
 else:
+    extra_compile = []
     _libs = []
 eci_kwds = dict(
     include_dirs = [SRC],
     includes = ['rvmprof.h', 'vmprof_stack.h'],
     libraries = _libs,
-    separate_module_files = [SRC.join('rvmprof.c')],
+    separate_module_files = [
+        SRC.join('rvmprof.c'),
+        SRC.join('compat.c'),
+        SRC.join('machine.c'),
+        SRC.join('symboltable.c'),
+        SRC.join('stack.c'),
+        # udis86
+        SRC.join('libudis86/decode.c'),
+        SRC.join('libudis86/itab.c'),
+        SRC.join('libudis86/udis86.c'),
+    ] + extra_compile,
     post_include_bits=['#define RPYTHON_VMPROF\n'],
     )
 global_eci = ExternalCompilationInfo(**eci_kwds)
diff --git a/rpython/rlib/rvmprof/src/_vmprof.h b/rpython/rlib/rvmprof/src/_vmprof.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/_vmprof.h
@@ -0,0 +1,75 @@
+#pragma once
+
+#include <Python.h>
+#include <frameobject.h>
+
+#ifdef VMPROF_WINDOWS
+#include "msiinttypes/inttypes.h"
+#include "msiinttypes/stdint.h"
+#else
+#include <inttypes.h>
+#include <stdint.h>
+#endif
+
+/**
+ * This whole setup is very strange. There was just one C file called
+ * _vmprof.c which included all *.h files to copy code. Unsure what
+ * the goal was with this design, but I assume it just 'GREW'
+ *
+ * Thus I'm (plan_rich) slowly trying to separate this. *.h files
+ * should not have complex implementations (all of them currently have them)
+ */
+
+
+#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int))
+
+#define ROUTINE_IS_PYTHON(RIP) ((unsigned long long)RIP & 0x1) == 0
+#define ROUTINE_IS_C(RIP) ((unsigned long long)RIP & 0x1) == 1
+
+typedef uint64_t ptr_t;
+
+/* This returns the address of the code object
+   as the identifier.  The mapping from identifiers to string
+   representations of the code object is done elsewhere, namely:
+
+   * If the code object dies while vmprof is enabled,
+     PyCode_Type.tp_dealloc will emit it.  (We don't handle nicely
+     for now the case where several code objects are created and die
+     at the same memory address.)
+
+   * When _vmprof.disable() is called, then we look around the
+     process for code objects and emit all the ones that we can
+     find (which we hope is very close to 100% of them).
+*/
+#define CODE_ADDR_TO_UID(co)  (((intptr_t)(co)))
+
+#define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0
+
+PyObject* vmprof_eval(PyFrameObject *f, int throwflag);
+
+#ifdef VMPROF_UNIX
+#define VMP_SUPPORTS_NATIVE_PROFILING
+#endif
+
+#define MARKER_STACKTRACE '\x01'
+#define MARKER_VIRTUAL_IP '\x02'
+#define MARKER_TRAILER '\x03'
+#define MARKER_INTERP_NAME '\x04'   /* deprecated */
+#define MARKER_HEADER '\x05'
+#define MARKER_TIME_N_ZONE '\x06'
+#define MARKER_META '\x07'
+#define MARKER_NATIVE_SYMBOLS '\x08'
+
+#define VERSION_BASE '\x00'
+#define VERSION_THREAD_ID '\x01'
+#define VERSION_TAG '\x02'
+#define VERSION_MEMORY '\x03'
+#define VERSION_MODE_AWARE '\x04'
+#define VERSION_DURATION '\x05'
+#define VERSION_TIMESTAMP '\x06'
+
+#define PROFILE_MEMORY '\x01'
+#define PROFILE_LINES  '\x02'
+#define PROFILE_NATIVE '\x04'
+
+int vmp_write_all(const char *buf, size_t bufsize);
diff --git a/rpython/rlib/rvmprof/src/compat.c b/rpython/rlib/rvmprof/src/compat.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/compat.c
@@ -0,0 +1,137 @@
+#include "compat.h"
+
+#include "_vmprof.h"
+
+#if VMPROF_WINDOWS
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#endif
+
+static int _vmp_profile_fileno = -1;
+
+int vmp_profile_fileno(void) {
+    return _vmp_profile_fileno;
+}
+void vmp_set_profile_fileno(int fileno) {
+    _vmp_profile_fileno = fileno;
+}
+
+#ifndef VMPROF_WINDOWS
+int vmp_write_all(const char *buf, size_t bufsize)
+{
+    ssize_t count;
+    if (_vmp_profile_fileno == -1) {
+        return -1;
+    }
+    while (bufsize > 0) {
+        count = write(_vmp_profile_fileno, buf, bufsize);
+        if (count <= 0)
+            return -1;   /* failed */
+        buf += count;
+        bufsize -= count;
+    }
+    return 0;
+}
+#endif
+
+int vmp_write_meta(const char * key, const char * value)
+{
+    char marker = MARKER_META;
+    long x = (long)strlen(key);
+    vmp_write_all(&marker, 1);
+    vmp_write_all((char*)&x, sizeof(long));
+    vmp_write_all(key, x);
+    x = (long)strlen(value);
+    vmp_write_all((char*)&x, sizeof(long));
+    vmp_write_all(value, x);
+    return 0;
+}
+
+/**
+ * Write the time and zone now.
+ */
+
+struct timezone_buf {
+    int64_t tv_sec;
+    int64_t tv_usec;
+};
+#define __SIZE (1+sizeof(struct timezone_buf)+8)
+
+#ifdef VMPROF_UNIX
+int vmp_write_time_now(int marker) {
+    char buffer[__SIZE];
+    struct timezone_buf buf;
+
+    (void)memset(&buffer, 0, __SIZE);
+
+    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
+           "marker must be either a trailer or time_n_zone!");
+
+    struct timeval tv;
+    time_t now;
+    struct tm tm;
+
+
+    /* copy over to the struct */
+    if (gettimeofday(&tv, NULL) != 0) {
+        return -1;
+    }
+    if (time(&now) == (time_t)-1) {
+        return -1;
+    }
+    if (localtime_r(&now, &tm) == NULL) {
+        return -1;
+    }
+    buf.tv_sec = tv.tv_sec;
+    buf.tv_usec = tv.tv_usec;
+    strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
+
+    buffer[0] = marker;
+    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
+    vmp_write_all(buffer, __SIZE);
+    return 0;
+}
+#endif
+
+#ifdef VMPROF_WINDOWS
+int vmp_write_time_now(int marker) {
+    char buffer[__SIZE];
+    struct timezone_buf buf;
+
+    /**
+     * http://stackoverflow.com/questions/10905892/equivalent-of-gettimeday-for-windows
+     */
+
+    // Note: some broken versions only have 8 trailing zero's, the correct
+    // epoch has 9 trailing zero's
+    static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL);
+
+    SYSTEMTIME  system_time;
+    FILETIME    file_time;
+    uint64_t    time;
+
+    (void)memset(&buffer, 0, __SIZE);
+
+    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
+           "marker must be either a trailer or time_n_zone!");
+
+
+    GetSystemTime( &system_time );
+    SystemTimeToFileTime( &system_time, &file_time );
+    time =  ((uint64_t)file_time.dwLowDateTime )      ;
+    time += ((uint64_t)file_time.dwHighDateTime) << 32;
+
+    buf.tv_sec = ((time - EPOCH) / 10000000L);
+    buf.tv_usec = (system_time.wMilliseconds * 1000);
+
+    // time zone not implemented on windows
+    memset(((char*)buffer)+__SIZE-8, 0, 8);
+    (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3);
+
+    buffer[0] = marker;
+    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
+    vmp_write_all(buffer, __SIZE);
+    return 0;
+}
+#endif
+#undef __SIZE
diff --git a/rpython/rlib/rvmprof/src/compat.h b/rpython/rlib/rvmprof/src/compat.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/compat.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include <Python.h>
+
+#if PY_MAJOR_VERSION >= 3
+    #define PyStr_AS_STRING PyBytes_AS_STRING
+    #define PyStr_GET_SIZE PyBytes_GET_SIZE
+    #define PyStr_NEW      PyUnicode_FromString
+    #define PyLong_NEW     PyLong_FromLong
+#else
+    #define PyStr_AS_STRING PyString_AS_STRING
+    #define PyStr_GET_SIZE PyString_GET_SIZE
+    #define PyStr_NEW      PyString_FromString
+    #define PyLong_NEW     PyInt_FromLong
+#endif
+
+int vmp_write_all(const char *buf, size_t bufsize);
+int vmp_write_time_now(int marker);
+int vmp_write_meta(const char * key, const char * value);
+
+int vmp_profile_fileno(void);
+void vmp_set_profile_fileno(int fileno);
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog b/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog
@@ -0,0 +1,602 @@
+2017-01-01  Jakub Jelinek  <jakub at redhat.com>
+
+	Update copyright years.
+
+2016-11-15  Matthias Klose  <doko at ubuntu.com>
+
+	* configure: Regenerate.
+
+2016-09-11  Carlos Liam  <carlos at aarzee.me>
+
+	* all: Remove meaningless trailing whitespace.
+
+2016-05-18  Uros Bizjak  <ubizjak at gmail.com>
+
+	PR target/71161
+	* elf.c (phdr_callback) [__i386__]: Add
+	__attribute__((__force_align_arg_pointer__)).
+
+2016-03-02  Maxim Ostapenko  <m.ostapenko at partner.samsung.com>
+
+	* elf.c (backtrace_initialize): Properly initialize elf_fileline_fn to
+	avoid possible crash.
+	(elf_add): Don't set *fileline_fn to elf_nodebug value in case of
+	missing debug info anymore.
+
+2016-02-06  John David Anglin  <danglin at gcc.gnu.org>
+
+	* mmap.c (MAP_FAILED): Define if not defined.
+
+2016-01-04  Jakub Jelinek  <jakub at redhat.com>
+
+	Update copyright years.
+
+2015-12-18  Andris Pavenis  <andris.pavenis at iki.fi>
+
+	* configure.ac: Specify that DJGPP do not have mmap
+	even when sys/mman.h exists.
+	* configure: Regenerate
+
+2015-12-09  John David Anglin  <danglin at gcc.gnu.org>
+
+	PR libgfortran/68115
+	* configure.ac: Set libbacktrace_cv_sys_sync to no on hppa*-*-hpux*.
+	* configure: Regenerate.
+	* elf.c (backtrace_initialize): Cast __sync_bool_compare_and_swap call
+	to void.
+
+2015-09-17  Ian Lance Taylor  <iant at google.com>
+
+	* posix.c (backtrace_open): Cast second argument of open() to int.
+
+2015-09-11  Ian Lance Taylor  <iant at google.com>
+
+	* Makefile.am (backtrace.lo): Depend on internal.h.
+	(sort.lo, stest.lo): Add explicit dependencies.
+	* Makefile.in: Rebuild.
+
+2015-09-09  Hans-Peter Nilsson  <hp at axis.com>
+
+	* backtrace.c: #include <sys/types.h>.
+
+2015-09-08  Ian Lance Taylor  <iant at google.com>
+
+	PR other/67457
+	* backtrace.c: #include "internal.h".
+	(struct backtrace_data): Add can_alloc field.
+	(unwind): If can_alloc is false, don't try to get file/line
+	information.
+	(backtrace_full): Set can_alloc field in bdata.
+	* alloc.c (backtrace_alloc): Don't call error_callback if it is
+	NULL.
+	* mmap.c (backtrace_alloc): Likewise.
+	* internal.h: Update comments for backtrace_alloc and
+	backtrace_free.
+
+2015-09-08  Ian Lance Taylor  <iant at google.com>
+
+	PR other/67457
+	* mmap.c (backtrace_alloc): Correct test for mmap failure.
+
+2015-08-31  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
+
+	* configure.ac: For spu-*-* targets, set have_fcntl to no.
+	* configure: Regenerate.
+
+2015-08-27  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
+
+	* configure.ac: Remove [disable-shared] argument to LT_INIT.
+	Remove setting PIC_FLAG when building as target library.
+	* configure: Regenerate.
+
+2015-08-26  Hans-Peter Nilsson  <hp at axis.com>
+
+	* configure.ac: Only compile with -fPIC if the target
+	supports it.
+	* configure: Regenerate.
+
+2015-08-24  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
+
+	* configure.ac: Set have_mmap to no on spu-*-* targets.
+	* configure: Regenerate.
+
+2015-08-13  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (read_function_entry): Add vec_inlined parameter.
+	Change all callers.
+
+2015-06-11  Martin Sebor  <msebor at redhat.com>
+
+	PR sanitizer/65479
+	* dwarf.c (struct line): Add new field idx.
+	(line_compare): Use it.
+	(add_line): Set it.
+	(read_line_info): Reset it.
+
+2015-05-29  Tristan Gingold  <gingold at adacore.com>
+
+	* pecoff.c: New file.
+	* Makefile.am (FORMAT_FILES): Add pecoff.c and dependencies.
+	* Makefile.in: Regenerate.
+	* filetype.awk: Detect pecoff.
+	* configure.ac: Define BACKTRACE_SUPPORTS_DATA on elf platforms.
+	Add pecoff.
+	* btest.c (test5): Test enabled only if BACKTRACE_SUPPORTS_DATA is
+	true.
+	* backtrace-supported.h.in (BACKTRACE_SUPPORTS_DATA): Define.
+	* configure: Regenerate.
+	* pecoff.c: New file.
+
+2015-05-13  Michael Haubenwallner  <michael.haubenwallner at ssi-schaefer.com>
+
+	* Makefile.in: Regenerated with automake-1.11.6.
+	* aclocal.m4: Likewise.
+	* configure: Likewise.
+
+2015-01-24  Matthias Klose  <doko at ubuntu.com>
+
+	* configure.ac: Move AM_ENABLE_MULTILIB before AC_PROG_CC.
+	* configure: Regenerate.
+
+2015-01-05  Jakub Jelinek  <jakub at redhat.com>
+
+	Update copyright years.
+
+2014-11-21  H.J. Lu  <hongjiu.lu at intel.com>
+
+	PR bootstrap/63784
+	* configure: Regenerated.
+
+2014-11-11  David Malcolm  <dmalcolm at redhat.com>
+
+	* ChangeLog.jit: New.
+
+2014-11-11  Francois-Xavier Coudert  <fxcoudert at gcc.gnu.org>
+
+	PR target/63610
+	* configure: Regenerate.
+
+2014-10-23  Ian Lance Taylor  <iant at google.com>
+
+	* internal.h (backtrace_atomic_load_pointer) [no atomic or sync]:
+	Fix to return void *.
+
+2014-05-08  Ian Lance Taylor  <iant at google.com>
+
+	* mmap.c (backtrace_free): If freeing a large aligned block of
+	memory, call munmap rather than holding onto it.
+	(backtrace_vector_grow): When growing a vector, double the number
+	of pages requested.  When releasing the old version of a grown
+	vector, pass the correct size to backtrace_free.
+
+2014-03-07  Ian Lance Taylor  <iant at google.com>
+
+	* sort.c (backtrace_qsort): Use middle element as pivot.
+
+2014-03-06  Ian Lance Taylor  <iant at google.com>
+
+	* sort.c: New file.
+	* stest.c: New file.
+	* internal.h (backtrace_qsort): Declare.
+	* dwarf.c (read_abbrevs): Call backtrace_qsort instead of qsort.
+	(read_line_info, read_function_entry): Likewise.
+	(read_function_info, build_dwarf_data): Likewise.
+	* elf.c (elf_initialize_syminfo): Likewise.
+	* Makefile.am (libbacktrace_la_SOURCES): Add sort.c.
+	(stest_SOURCES, stest_LDADD): Define.
+	(check_PROGRAMS): Add stest.
+
+2014-02-07  Misty De Meo  <misty at brew.sh>
+
+	PR target/58710
+	* configure.ac: Use AC_LINK_IFELSE in check for
+	_Unwind_GetIPInfo.
+	* configure: Regenerate.
+
+2014-01-02  Richard Sandiford  <rdsandiford at googlemail.com>
+
+	Update copyright years
+
+2013-12-06  Jakub Jelinek  <jakub at redhat.com>
+
+	* elf.c (ET_DYN): Undefine and define again.
+	(elf_add): Add exe argument, if true and ehdr.e_type is ET_DYN,
+	return early -1 without closing the descriptor.
+	(struct phdr_data): Add exe_descriptor.
+	(phdr_callback): If pd->exe_descriptor is not -1, for very first
+	call if dlpi_name is NULL just call elf_add with the exe_descriptor,
+	otherwise backtrace_close the exe_descriptor if not -1.  Adjust
+	call to elf_add.
+	(backtrace_initialize): Adjust call to elf_add.  If it returns
+	-1, set pd.exe_descriptor to descriptor, otherwise set it to -1.
+
+2013-12-05  Ian Lance Taylor  <iant at google.com>
+
+	* alloc.c (backtrace_vector_finish): Add error_callback and data
+	parameters.  Call backtrace_vector_release.  Return address base.
+	* mmap.c (backtrace_vector_finish): Add error_callback and data
+	parameters.  Return address base.
+	* dwarf.c (read_function_info): Get new address base from
+	backtrace_vector_finish.
+	* internal.h (backtrace_vector_finish): Update declaration.
+
+2013-11-27  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (find_address_ranges): New static function, broken out
+	of build_address_map.
+	(build_address_map): Call it.
+	* btest.c (check): Check for missing filename or function, rather
+	than crashing.
+	(f3): Check that enough frames were returned.
+
+2013-11-19  Jakub Jelinek  <jakub at redhat.com>
+
+	* backtrace.h (backtrace_syminfo_callback): Add symsize argument.
+	* elf.c (elf_syminfo): Pass 0 or sym->size to the callback as
+	last argument.
+	* btest.c (struct symdata): Add size field.
+	(callback_three): Add symsize argument.  Copy it to the data->size
+	field.
+	(f23): Set symdata.size to 0.
+	(test5): Likewise.  If sizeof (int) > 1, lookup address of
+	((uintptr_t) &global) + 1.  Verify symdata.val and symdata.size
+	values.
+
+	* atomic.c: Include sys/types.h.
+
+2013-11-18  Ian Lance Taylor  <iant at google.com>
+
+	* configure.ac: Check for support of __atomic extensions.
+	* internal.h: Declare or #define atomic functions for use in
+	backtrace code.
+	* atomic.c: New file.
+	* dwarf.c (dwarf_lookup_pc): Use atomic functions.
+	(dwarf_fileline, backtrace_dwarf_add): Likewise.
+	* elf.c (elf_add_syminfo_data, elf_syminfo): Likewise.
+	(backtrace_initialize): Likewise.
+	* fileline.c (fileline_initialize): Likewise.
+	* Makefile.am (libbacktrace_la_SOURCES): Add atomic.c.
+	* configure, config.h.in, Makefile.in: Rebuild.
+
+2013-11-18  Jakub Jelinek  <jakub at redhat.com>
+
+	* elf.c (SHN_UNDEF): Define.
+	(elf_initialize_syminfo): Add base_address argument.  Ignore symbols
+	with st_shndx == SHN_UNDEF.  Add base_address to address fields.
+	(elf_add): Adjust caller.
+
+	* elf.c (phdr_callback): Process info->dlpi_addr == 0 normally.
+
+2013-11-16  Ian Lance Taylor  <iant at google.com>
+
+	* backtrace.h (backtrace_create_state): Correct comment about
+	threading.
+
+2013-11-15  Ian Lance Taylor  <iant at google.com>
+
+	* backtrace.h (backtrace_syminfo): Update comment and parameter
+	name to take any address, not just a PC value.
+	* elf.c (STT_OBJECT): Define.
+	(elf_nosyms): Rename parameter pc to addr.
+	(elf_symbol_search): Rename local variable pc to addr.
+	(elf_initialize_syminfo): Add STT_OBJECT symbols to elf_symbols.
+	(elf_syminfo): Rename parameter pc to addr.
+	* btest.c (global): New global variable.
+	(test5): New test.
+	(main): Call test5.
+
+2013-10-17  Ian Lance Taylor  <iant at google.com>
+
+	* elf.c (elf_add): Don't get the wrong offsets if a debug section
+	is missing.
+
+2013-10-15  David Malcolm  <dmalcolm at redhat.com>
+
+	* configure.ac: Add --enable-host-shared, setting up
+	pre-existing PIC_FLAG variable within Makefile.am et al.
+	* configure: Regenerate.
+
+2013-09-20  Alan Modra  <amodra at gmail.com>
+
+	* configure: Regenerate.
+
+2013-07-23  Alexander Monakov  <amonakov at ispras.ru>
+
+	* elf.c (elf_syminfo): Loop over the elf_syminfo_data chain.
+
+2013-07-23  Alexander Monakov  <amonakov at ispras.ru>
+
+	* elf.c (backtrace_initialize): Pass elf_fileline_fn to
+	dl_iterate_phdr callbacks.
+
+2013-03-25  Ian Lance Taylor  <iant at google.com>
+
+	* alloc.c: #include <sys/types.h>.
+	* mmap.c: Likewise.
+
+2013-01-31  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (read_function_info): Permit fvec parameter to be NULL.
+	(dwarf_lookup_pc): Don't use ddata->fvec if threaded.
+
+2013-01-25  Jakub Jelinek  <jakub at redhat.com>
+
+	PR other/56076
+	* dwarf.c (read_line_header): Don't crash if DW_AT_comp_dir
+	attribute was not seen.
+
+2013-01-16  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (struct unit): Add filename and abs_filename fields.
+	(build_address_map): Set new fields when reading unit.
+	(dwarf_lookup_pc): If we don't find an entry in the line table,
+	just return the main file name.
+
+2013-01-14  Richard Sandiford  <rdsandiford at googlemail.com>
+
+	Update copyright years.
+
+2013-01-01  Ian Lance Taylor  <iant at google.com>
+
+	PR bootstrap/54834
+	* Makefile.am (AM_CPPFLAGS): Remove -I ../gcc/include and -I
+	$(MULTIBUILDTOP)/../../gcc/include.
+	* Makefile.in: Rebuild.
+
+2013-01-01  Ian Lance Taylor  <iant at google.com>
+
+	PR other/55536
+	* mmap.c (backtrace_alloc): Don't call sync functions if not
+	threaded.
+	(backtrace_free): Likewise.
+
+2012-12-12  John David Anglin  <dave.anglin at nrc-cnrc.gc.ca>
+
+	* mmapio.c: Define MAP_FAILED if not defined.
+
+2012-12-11  Jakub Jelinek  <jakub at redhat.com>
+
+	PR bootstrap/54926
+	* Makefile.am (AM_CFLAGS): Remove -frandom-seed=$@.
+	* configure.ac: If --with-target-subdir, add -frandom-seed=$@
+	to EXTRA_FLAGS unconditionally, otherwise check whether the compiler
+	accepts it.
+	* Makefile.in: Regenerated.
+	* configure: Regenerated.
+
+2012-12-07  Jakub Jelinek  <jakub at redhat.com>
+
+	PR bootstrap/54926
+	* Makefile.am (AM_CFLAGS): Add -frandom-seed=$@.
+	* Makefile.in: Regenerated.
+
+2012-11-20  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (read_attribute): Always clear val.
+
+2012-11-13  Ian Lance Taylor  <iant at google.com>
+
+	PR other/55312
+	* configure.ac: Only add -Werror if building a target library.
+	* configure: Rebuild.
+
+2012-11-12  Ian Lance Taylor  <iant at google.com>
+	    Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
+	    Gerald Pfeifer  <gerald at pfeifer.com>
+
+	* configure.ac: Check for getexecname.
+	* fileline.c: #include <errno.h>.  Define getexecname if not
+	available.
+	(fileline_initialize): Try to find the executable in a few
+	different ways.
+	* print.c (error_callback): Only print the filename if it came
+	from the backtrace state.
+	* configure, config.h.in: Rebuild.
+
+2012-10-29  Ian Lance Taylor  <iant at google.com>
+
+	* mmap.c (backtrace_vector_release): Correct last patch: add
+	aligned, not size.
+
+2012-10-29  Ian Lance Taylor  <iant at google.com>
+
+	* mmap.c (backtrace_vector_release): Make sure freed block is
+	aligned on 8-byte boundary.
+
+2012-10-26  Ian Lance Taylor  <iant at google.com>
+
+	PR other/55087
+	* posix.c (backtrace_open): Add does_not_exist parameter.
+	* elf.c (phdr_callback): Do not warn if shared library could not
+	be opened.
+	* fileline.c (fileline_initialize): Update calls to
+	backtrace_open.
+	* internal.h (backtrace_open): Update declaration.
+
+2012-10-26  Jack Howarth  <howarth at bromo.med.uc.edu>
+
+	PR target/55061
+	* configure.ac: Check for _Unwind_GetIPInfo function declaration.
+	* configure: Regenerate.
+
+2012-10-24  Ian Lance Taylor  <iant at google.com>
+
+	PR target/55061
+	* configure.ac: Check whether -funwind-tables option works.
+	* configure: Rebuild.
+
+2012-10-11  Ian Lance Taylor  <iant at google.com>
+
+	* configure.ac: Do not use dl_iterate_phdr on Solaris 10.
+	* configure: Rebuild.
+
+2012-10-10  Ian Lance Taylor  <iant at google.com>
+
+	* elf.c: Rename all Elf typedefs to start with b_elf, and be all
+	lower case.
+
+2012-10-10  Hans-Peter Nilsson  <hp at bitrange.com>
+
+	* elf.c (elf_add_syminfo_data): Add casts to avoid warning.
+
+2012-10-09  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (dwarf_fileline): Add cast to avoid warning.
+	(backtrace_dwarf_add): Likewise.
+
+2012-10-09  Ian Lance Taylor  <iant at google.com>
+
+	Add support for tracing through shared libraries.
+	* configure.ac: Check for link.h and dl_iterate_phdr.
+	* elf.c: #include <link.h> if system has dl_iterate_phdr.  #undef
+	ELF macros before #defining them.
+	(dl_phdr_info, dl_iterate_phdr): Define if system does not have
+	dl_iterate_phdr.
+	(struct elf_syminfo_data): Add next field.
+	(elf_initialize_syminfo): Initialize next field.
+	(elf_add_syminfo_data): New static function.
+	(elf_add): New static function, broken out of
+	backtrace_initialize.  Call backtrace_dwarf_add instead of
+	backtrace_dwarf_initialize.
+	(struct phdr_data): Define.
+	(phdr_callback): New static function.
+	(backtrace_initialize): Call elf_add.
+	* dwarf.c (struct dwarf_data): Add next and base_address fields.
+	(add_unit_addr): Add base_address parameter.  Change all callers.
+	(add_unit_ranges, build_address_map): Likewise.
+	(add_line): Add ddata parameter.  Change all callers.
+	(read_line_program, add_function_range): Likewise.
+	(dwarf_lookup_pc): New static function, broken out of
+	dwarf_fileline.
+	(dwarf_fileline): Call dwarf_lookup_pc.
+	(build_dwarf_data): New static function.
+	(backtrace_dwarf_add): New function.
+	(backtrace_dwarf_initialize): Remove.
+	* internal.h (backtrace_dwarf_initialize): Don't declare.
+	(backtrace_dwarf_add): Declare.
+	* configure, config.h.in: Rebuild.
+
+2012-10-04  Gerald Pfeifer  <gerald at pfeifer.com>
+
+	* btest.c (f23): Avoid uninitialized variable warning.
+
+2012-10-04  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c: If the system header files do not declare strnlen,
+	provide our own version.
+
+2012-10-03  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (read_uleb128): Fix overflow test.
+	(read_sleb128): Likewise.
+	(build_address_map): Don't change unit_buf.start.
+
+2012-10-02  Uros Bizjak  <ubizjak at gmail.com>
+
+	PR other/54761
+	* configure.ac (EXTRA_FLAGS): New.
+	* Makefile.am (AM_FLAGS): Add $(EXTRA_FLAGS).
+	* configure, Makefile.in: Regenerate.
+
+2012-09-29  Ian Lance Taylor  <iant at google.com>
+
+	PR other/54749
+	* fileline.c (fileline_initialize): Pass errnum as -1 when
+	reporting that we could not read executable information after a
+	previous failure.
+
+2012-09-27  Ian Lance Taylor  <iant at google.com>
+
+	PR bootstrap/54732
+	* configure.ac: Add no-dependencies to AM_INIT_AUTOMAKE.
+	* Makefile.am: Add dependencies for all objects.
+	* configure, aclocal.m4, Makefile.in: Rebuild.
+
+2012-09-27  Ian Lance Taylor  <iant at google.com>
+
+	PR other/54726
+	* elf.c (backtrace_initialize): Set *fileln_fn, not
+	state->fileln_fn.
+
+2012-09-19  Ian Lance Taylor  <iant at google.com>
+
+	* configure.ac: Only use GCC_CHECK_UNWIND_GETIPINFO when compiled
+	as a target library.
+	* configure: Rebuild.
+
+2012-09-19  Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
+	    Ian Lance Taylor  <iant at google.com>
+
+        * configure.ac (GCC_HEADER_STDINT): Invoke.
+        * backtrace.h: If we can't find <stdint.h>, use "gstdint.h".
+        * btest.c: Don't include <stdint.h>.
+        * dwarf.c: Likewise.
+        * configure, aclocal.m4, Makefile.in, config.h.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  <iant at google.com>
+
+	PR bootstrap/54623
+	* Makefile.am (AM_CPPFLAGS): Define.
+	(AM_CFLAGS): Remove -I options.
+	* Makefile.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  <iant at google.com>
+
+	* posix.c (O_BINARY): Define if not defined.
+	(backtrace_open): Pass O_BINARY to open.  Only call fcntl if
+	HAVE_FCNTL is defined.
+	* configure.ac: Test for the fcntl function.
+	* configure, config.h.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  <iant at google.com>
+
+	* btest.c (test1, test2, test3, test4): Add the unused attribute.
+
+2012-09-18  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c: Correct test of HAVE_DECL_STRNLEN.
+
+2012-09-18  Ian Lance Taylor  <iant at google.com>
+
+	* configure.ac: Add AC_USE_SYSTEM_EXTENSIONS.
+	* mmapio.c: Don't define _GNU_SOURCE.
+	* configure, config.h.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  <iant at google.com>
+
+	* configure.ac: Check whether strnlen is declared.
+	* dwarf.c: Declare strnlen if not declared.
+	* configure, config.h.in: Rebuild.
+
+2012-09-18  Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
+
+	* fileline.c: Include <stdlib.h>.
+	* mmap.c: Likewise.
+
+2012-09-17  Ian Lance Taylor  <iant at google.com>
+
+	PR bootstrap/54611
+	* nounwind.c (backtrace_full): Rename from backtrace.  Add state
+	parameter.
+
+2012-09-17  Gerald Pfeifer  <gerald at pfeifer.com>
+
+	PR bootstrap/54611
+	* nounwind.c (backtrace_simple): Add state parameter.
+
+2012-09-17  Ian Lance Taylor  <iant at google.com>
+
+	PR bootstrap/54609
+	* unknown.c (unknown_fileline): Add state parameter, remove
+	fileline_data parameter, name error_callback parameter.
+	(backtrace_initialize): Add state parameter.
+
+2012-09-17  Ian Lance Taylor  <iant at google.com>
+
+	* Initial implementation.
+

+Copyright (C) 2012-2017 Free Software Foundation, Inc.
+
+Copying and distribution of this file, with or without modification,
+are permitted in any medium without royalty provided the copyright
+notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog.jit b/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog.jit
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog.jit
@@ -0,0 +1,14 @@
+2014-09-24  David Malcolm  <dmalcolm at redhat.com>
+
+	* ChangeLog.jit: Add copyright footer.
+
+2013-10-03  David Malcolm  <dmalcolm at redhat.com>
+
+	* configure.ac: Add --enable-host-shared.
+	* configure: Regenerate.
+

+Copyright (C) 2013-2014 Free Software Foundation, Inc.
+
+Copying and distribution of this file, with or without modification,
+are permitted in any medium without royalty provided the copyright
+notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/Makefile b/rpython/rlib/rvmprof/src/libbacktrace/Makefile
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/libbacktrace/Makefile
@@ -0,0 +1,770 @@
+# Makefile.in generated by automake 1.11.6 from Makefile.am.
+# Makefile.  Generated from Makefile.in by configure.
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+# Foundation, Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+
+
+# Makefile.am -- Backtrace Makefile.
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+#     (1) Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+
+#     (2) Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+
+#     (3) The name of the author may not be used to
+#     endorse or promote products derived from this software without
+#     specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+
+am__make_dryrun = \
+  { \
+    am__dry=no; \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        echo 'am--echo: ; @echo "AM"  OK' | $(MAKE) -f - 2>/dev/null \
+          | grep '^AM OK$$' >/dev/null || am__dry=yes;; \
+      *) \
+        for am__flg in $$MAKEFLAGS; do \
+          case $$am__flg in \
+            *=*|--*) ;; \
+            *n*) am__dry=yes; break;; \
+          esac; \
+        done;; \
+    esac; \
+    test $$am__dry = yes; \
+  }
+pkgdatadir = $(datadir)/libbacktrace
+pkgincludedir = $(includedir)/libbacktrace
+pkglibdir = $(libdir)/libbacktrace
+pkglibexecdir = $(libexecdir)/libbacktrace
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = x86_64-pc-linux-gnu
+host_triplet = x86_64-pc-linux-gnu
+target_triplet = x86_64-pc-linux-gnu
+check_PROGRAMS = $(am__EXEEXT_1)
+am__append_1 = btest stest
+subdir = .
+DIST_COMMON = README ChangeLog $(srcdir)/Makefile.in \
+	$(srcdir)/Makefile.am $(top_srcdir)/configure \
+	$(am__configure_deps) $(srcdir)/config.h.in \
+	$(srcdir)/../mkinstalldirs $(srcdir)/backtrace-supported.h.in
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/../config/lead-dot.m4 \
+	$(top_srcdir)/../config/multi.m4 \
+	$(top_srcdir)/../config/override.m4 \
+	$(top_srcdir)/../config/stdint.m4 \
+	$(top_srcdir)/../config/unwind_ipinfo.m4 \
+	$(top_srcdir)/../config/warnings.m4 \
+	$(top_srcdir)/../libtool.m4 $(top_srcdir)/../ltoptions.m4 \
+	$(top_srcdir)/../ltsugar.m4 $(top_srcdir)/../ltversion.m4 \
+	$(top_srcdir)/../lt~obsolete.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
+ configure.lineno config.status.lineno
+mkinstalldirs = $(SHELL) $(top_srcdir)/../mkinstalldirs
+CONFIG_HEADER = config.h
+CONFIG_CLEAN_FILES = backtrace-supported.h
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+am__DEPENDENCIES_1 =
+am_libbacktrace_la_OBJECTS = atomic.lo dwarf.lo fileline.lo posix.lo \
+	print.lo sort.lo state.lo
+libbacktrace_la_OBJECTS = $(am_libbacktrace_la_OBJECTS)
+am__EXEEXT_1 = btest$(EXEEXT) stest$(EXEEXT)
+am_btest_OBJECTS = btest-btest.$(OBJEXT)
+btest_OBJECTS = $(am_btest_OBJECTS)
+btest_DEPENDENCIES = libbacktrace.la
+btest_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=link $(CCLD) $(btest_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
+	$(LDFLAGS) -o $@
+am_stest_OBJECTS = stest.$(OBJEXT)
+stest_OBJECTS = $(am_stest_OBJECTS)
+stest_DEPENDENCIES = libbacktrace.la
+DEFAULT_INCLUDES = -I.
+depcomp =
+am__depfiles_maybe =
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+CCLD = $(CC)
+LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
+	$(LDFLAGS) -o $@
+SOURCES = $(libbacktrace_la_SOURCES) $(EXTRA_libbacktrace_la_SOURCES) \
+	$(btest_SOURCES) $(stest_SOURCES)
+MULTISRCTOP = 
+MULTIBUILDTOP = 
+MULTIDIRS = 
+MULTISUBDIR = 
+MULTIDO = true
+MULTICLEAN = true
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+ETAGS = etags
+CTAGS = ctags
+am__tty_colors = \
+red=; grn=; lgn=; blu=; std=
+ACLOCAL = ${SHELL} /home/rich/src/gcc/missing --run aclocal-1.11
+ALLOC_FILE = mmap.lo
+AMTAR = $${TAR-tar}
+AR = ar
+AUTOCONF = ${SHELL} /home/rich/src/gcc/missing --run autoconf
+AUTOHEADER = ${SHELL} /home/rich/src/gcc/missing --run autoheader
+AUTOMAKE = ${SHELL} /home/rich/src/gcc/missing --run automake-1.11
+AWK = gawk
+BACKTRACE_FILE = backtrace.lo simple.lo
+BACKTRACE_SUPPORTED = 1
+BACKTRACE_SUPPORTS_DATA = 1
+BACKTRACE_SUPPORTS_THREADS = 1
+BACKTRACE_USES_MALLOC = 0
+CC = gcc
+CFLAGS = -g -O2
+CPP = gcc -E
+CPPFLAGS = 
+CYGPATH_W = echo
+DEFS = -DHAVE_CONFIG_H
+DSYMUTIL = 
+DUMPBIN = 
+ECHO_C = 
+ECHO_N = -n
+ECHO_T = 
+EGREP = /usr/bin/grep -E
+EXEEXT = 
+EXTRA_FLAGS = -funwind-tables -frandom-seed=$@
+FGREP = /usr/bin/grep -F
+FORMAT_FILE = elf.lo
+GREP = /usr/bin/grep
+INSTALL = /usr/bin/install -c
+INSTALL_DATA = ${INSTALL} -m 644
+INSTALL_PROGRAM = ${INSTALL}
+INSTALL_SCRIPT = ${INSTALL}
+INSTALL_STRIP_PROGRAM = $(install_sh) -c -s
+LD = /usr/bin/ld -m elf_x86_64
+LDFLAGS = 
+LIBOBJS = 
+LIBS = 
+LIBTOOL = $(SHELL) $(top_builddir)/libtool
+LIPO = 
+LN_S = ln -s
+LTLIBOBJS = 
+MAINT = #
+MAKEINFO = ${SHELL} /home/rich/src/gcc/missing --run makeinfo
+MKDIR_P = /usr/bin/mkdir -p
+NM = /usr/bin/nm -B
+NMEDIT = 
+OBJDUMP = objdump
+OBJEXT = o
+OTOOL = 
+OTOOL64 = 
+PACKAGE = libbacktrace
+PACKAGE_BUGREPORT = 
+PACKAGE_NAME = package-unused
+PACKAGE_STRING = package-unused version-unused
+PACKAGE_TARNAME = libbacktrace
+PACKAGE_URL = 
+PACKAGE_VERSION = version-unused
+PATH_SEPARATOR = :
+PIC_FLAG = 
+RANLIB = ranlib
+SED = /usr/bin/sed
+SET_MAKE = 
+SHELL = /bin/sh
+STRIP = strip
+VERSION = version-unused
+VIEW_FILE = mmapio.lo
+WARN_FLAGS = -W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual
+abs_builddir = /home/rich/src/gcc/libbacktrace
+abs_srcdir = /home/rich/src/gcc/libbacktrace
+abs_top_builddir = /home/rich/src/gcc/libbacktrace
+abs_top_srcdir = /home/rich/src/gcc/libbacktrace
+ac_ct_CC = gcc
+ac_ct_DUMPBIN = 
+am__leading_dot = .
+am__tar = $${TAR-tar} chof - "$$tardir"
+am__untar = $${TAR-tar} xf -
+bindir = ${exec_prefix}/bin
+build = x86_64-pc-linux-gnu
+build_alias = 
+build_cpu = x86_64
+build_os = linux-gnu
+build_vendor = pc
+builddir = .
+datadir = ${datarootdir}
+datarootdir = ${prefix}/share
+docdir = ${datarootdir}/doc/${PACKAGE_TARNAME}
+dvidir = ${docdir}
+exec_prefix = ${prefix}
+host = x86_64-pc-linux-gnu
+host_alias = 
+host_cpu = x86_64
+host_os = linux-gnu
+host_vendor = pc
+htmldir = ${docdir}
+includedir = ${prefix}/include
+infodir = ${datarootdir}/info
+install_sh = ${SHELL} /home/rich/src/gcc/install-sh
+libdir = ${exec_prefix}/lib
+libexecdir = ${exec_prefix}/libexec
+libtool_VERSION = 1:0:0
+localedir = ${datarootdir}/locale
+localstatedir = ${prefix}/var
+mandir = ${datarootdir}/man
+mkdir_p = /usr/bin/mkdir -p
+multi_basedir = 
+oldincludedir = /usr/include
+pdfdir = ${docdir}
+prefix = /usr/local
+program_transform_name = s,x,x,
+psdir = ${docdir}
+sbindir = ${exec_prefix}/sbin
+sharedstatedir = ${prefix}/com
+srcdir = .
+sysconfdir = ${prefix}/etc
+target = x86_64-pc-linux-gnu
+target_alias = 
+target_cpu = x86_64
+target_os = linux-gnu
+target_vendor = pc
+top_build_prefix = 
+top_builddir = .
+top_srcdir = .
+ACLOCAL_AMFLAGS = -I .. -I ../config
+AM_CPPFLAGS = -I $(top_srcdir)/../include -I $(top_srcdir)/../libgcc \
+	-I ../libgcc
+
+AM_CFLAGS = $(EXTRA_FLAGS) $(WARN_FLAGS) $(PIC_FLAG)
+noinst_LTLIBRARIES = libbacktrace.la
+libbacktrace_la_SOURCES = \
+	backtrace.h \
+	atomic.c \
+	dwarf.c \
+	fileline.c \
+	internal.h \
+	posix.c \
+	print.c \
+	sort.c \
+	state.c
+
+BACKTRACE_FILES = \
+	backtrace.c \
+	simple.c \
+	nounwind.c
+
+FORMAT_FILES = \
+	elf.c \
+	pecoff.c \
+	unknown.c
+
+VIEW_FILES = \
+	read.c \
+	mmapio.c
+
+ALLOC_FILES = \
+	alloc.c \
+	mmap.c
+
+EXTRA_libbacktrace_la_SOURCES = \
+	$(BACKTRACE_FILES) \
+	$(FORMAT_FILES) \
+	$(VIEW_FILES) \
+	$(ALLOC_FILES)
+
+libbacktrace_la_LIBADD = \
+	$(BACKTRACE_FILE) \
+	$(FORMAT_FILE) \
+	$(VIEW_FILE) \
+	$(ALLOC_FILE)
+
+libbacktrace_la_DEPENDENCIES = $(libbacktrace_la_LIBADD)
+TESTS = $(check_PROGRAMS)
+btest_SOURCES = btest.c
+btest_CFLAGS = $(AM_CFLAGS) -g -O
+btest_LDADD = libbacktrace.la
+stest_SOURCES = stest.c
+stest_LDADD = libbacktrace.la
+
+# We can't use automake's automatic dependency tracking, because it
+# breaks when using bootstrap-lean.  Automatic dependency tracking
+# with GCC bootstrap will cause some of the objects to depend on
+# header files in prev-gcc/include, e.g., stddef.h and stdarg.h.  When
+# using bootstrap-lean, prev-gcc is removed after each stage.  When
+# running "make install", those header files will be gone, causing the
+# library to be rebuilt at install time.  That may not succeed.
+
+# These manual dependencies do not include dependencies on unwind.h,
+# even though that is part of GCC, because where to find it depends on
+# whether we are being built as a host library or a target library.
+INCDIR = $(top_srcdir)/../include
+all: config.h
+	$(MAKE) $(AM_MAKEFLAGS) all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+am--refresh: Makefile
+	@:
+$(srcdir)/Makefile.in: # $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      echo ' cd $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps'; \
+	      $(am__cd) $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps \
+		&& exit 0; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign --ignore-deps Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign --ignore-deps Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    echo ' $(SHELL) ./config.status'; \
+	    $(SHELL) ./config.status;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	$(SHELL) ./config.status --recheck
+
+$(top_srcdir)/configure: # $(am__configure_deps)
+	$(am__cd) $(srcdir) && $(AUTOCONF)
+$(ACLOCAL_M4): # $(am__aclocal_m4_deps)
+	$(am__cd) $(srcdir) && $(ACLOCAL) $(ACLOCAL_AMFLAGS)
+$(am__aclocal_m4_deps):
+
+config.h: stamp-h1
+	@if test ! -f $@; then rm -f stamp-h1; else :; fi
+	@if test ! -f $@; then $(MAKE) $(AM_MAKEFLAGS) stamp-h1; else :; fi
+
+stamp-h1: $(srcdir)/config.h.in $(top_builddir)/config.status
+	@rm -f stamp-h1
+	cd $(top_builddir) && $(SHELL) ./config.status config.h
+$(srcdir)/config.h.in: # $(am__configure_deps) 
+	($(am__cd) $(top_srcdir) && $(AUTOHEADER))
+	rm -f stamp-h1
+	touch $@
+
+distclean-hdr:
+	-rm -f config.h stamp-h1
+backtrace-supported.h: $(top_builddir)/config.status $(srcdir)/backtrace-supported.h.in
+	cd $(top_builddir) && $(SHELL) ./config.status $@
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; for p in $$list; do \
+	  dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
+	  test "$$dir" != "$$p" || dir=.; \
+	  echo "rm -f \"$${dir}/so_locations\""; \
+	  rm -f "$${dir}/so_locations"; \
+	done
+libbacktrace.la: $(libbacktrace_la_OBJECTS) $(libbacktrace_la_DEPENDENCIES) $(EXTRA_libbacktrace_la_DEPENDENCIES) 
+	$(LINK)  $(libbacktrace_la_OBJECTS) $(libbacktrace_la_LIBADD) $(LIBS)
+
+clean-checkPROGRAMS:
+	@list='$(check_PROGRAMS)'; test -n "$$list" || exit 0; \
+	echo " rm -f" $$list; \
+	rm -f $$list || exit $$?; \
+	test -n "$(EXEEXT)" || exit 0; \
+	list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+	echo " rm -f" $$list; \
+	rm -f $$list
+btest$(EXEEXT): $(btest_OBJECTS) $(btest_DEPENDENCIES) $(EXTRA_btest_DEPENDENCIES) 
+	@rm -f btest$(EXEEXT)
+	$(btest_LINK) $(btest_OBJECTS) $(btest_LDADD) $(LIBS)
+stest$(EXEEXT): $(stest_OBJECTS) $(stest_DEPENDENCIES) $(EXTRA_stest_DEPENDENCIES) 
+	@rm -f stest$(EXEEXT)
+	$(LINK) $(stest_OBJECTS) $(stest_LDADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+.c.o:
+	$(COMPILE) -c $<
+
+.c.obj:
+	$(COMPILE) -c `$(CYGPATH_W) '$<'`
+
+.c.lo:
+	$(LTCOMPILE) -c -o $@ $<
+
+btest-btest.o: btest.c
+	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(btest_CFLAGS) $(CFLAGS) -c -o btest-btest.o `test -f 'btest.c' || echo '$(srcdir)/'`btest.c
+
+btest-btest.obj: btest.c
+	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(btest_CFLAGS) $(CFLAGS) -c -o btest-btest.obj `if test -f 'btest.c'; then $(CYGPATH_W) 'btest.c'; else $(CYGPATH_W) '$(srcdir)/btest.c'; fi`
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+
+distclean-libtool:
+	-rm -f libtool config.lt
+
+# GNU Make needs to see an explicit $(MAKE) variable in the command it
+# runs to enable its job server during parallel builds.  Hence the
+# comments below.
+all-multi:
+	$(MULTIDO) $(AM_MAKEFLAGS) DO=all multi-do # $(MAKE)
+install-multi:
+	$(MULTIDO) $(AM_MAKEFLAGS) DO=install multi-do # $(MAKE)
+
+mostlyclean-multi:
+	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=mostlyclean multi-clean # $(MAKE)
+clean-multi:
+	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=clean multi-clean # $(MAKE)
+distclean-multi:
+	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=distclean multi-clean # $(MAKE)
+maintainer-clean-multi:
+	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=maintainer-clean multi-clean # $(MAKE)
+
+ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
+	list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
+	unique=`for i in $$list; do \
+	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+	  done | \
+	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+	      END { if (nonempty) { for (i in files) print i; }; }'`; \
+	mkid -fID $$unique
+tags: TAGS
+
+TAGS:  $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
+		$(TAGS_FILES) $(LISP)
+	set x; \
+	here=`pwd`; \
+	list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
+	unique=`for i in $$list; do \
+	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+	  done | \
+	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+	      END { if (nonempty) { for (i in files) print i; }; }'`; \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: CTAGS
+CTAGS:  $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
+		$(TAGS_FILES) $(LISP)
+	list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
+	unique=`for i in $$list; do \
+	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+	  done | \
+	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+	      END { if (nonempty) { for (i in files) print i; }; }'`; \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+check-TESTS: $(TESTS)
+	@failed=0; all=0; xfail=0; xpass=0; skip=0; \
+	srcdir=$(srcdir); export srcdir; \
+	list=' $(TESTS) '; \
+	$(am__tty_colors); \
+	if test -n "$$list"; then \
+	  for tst in $$list; do \
+	    if test -f ./$$tst; then dir=./; \
+	    elif test -f $$tst; then dir=; \
+	    else dir="$(srcdir)/"; fi; \
+	    if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
+	      all=`expr $$all + 1`; \
+	      case " $(XFAIL_TESTS) " in \
+	      *[\ \	]$$tst[\ \	]*) \
+		xpass=`expr $$xpass + 1`; \
+		failed=`expr $$failed + 1`; \
+		col=$$red; res=XPASS; \
+	      ;; \
+	      *) \
+		col=$$grn; res=PASS; \
+	      ;; \
+	      esac; \
+	    elif test $$? -ne 77; then \
+	      all=`expr $$all + 1`; \
+	      case " $(XFAIL_TESTS) " in \
+	      *[\ \	]$$tst[\ \	]*) \
+		xfail=`expr $$xfail + 1`; \
+		col=$$lgn; res=XFAIL; \
+	      ;; \
+	      *) \
+		failed=`expr $$failed + 1`; \
+		col=$$red; res=FAIL; \
+	      ;; \
+	      esac; \
+	    else \
+	      skip=`expr $$skip + 1`; \
+	      col=$$blu; res=SKIP; \
+	    fi; \
+	    echo "$${col}$$res$${std}: $$tst"; \
+	  done; \
+	  if test "$$all" -eq 1; then \
+	    tests="test"; \
+	    All=""; \
+	  else \
+	    tests="tests"; \
+	    All="All "; \
+	  fi; \
+	  if test "$$failed" -eq 0; then \
+	    if test "$$xfail" -eq 0; then \
+	      banner="$$All$$all $$tests passed"; \
+	    else \
+	      if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \
+	      banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \
+	    fi; \
+	  else \
+	    if test "$$xpass" -eq 0; then \
+	      banner="$$failed of $$all $$tests failed"; \
+	    else \
+	      if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \
+	      banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \
+	    fi; \
+	  fi; \
+	  dashes="$$banner"; \
+	  skipped=""; \
+	  if test "$$skip" -ne 0; then \
+	    if test "$$skip" -eq 1; then \
+	      skipped="($$skip test was not run)"; \
+	    else \
+	      skipped="($$skip tests were not run)"; \
+	    fi; \
+	    test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \
+	      dashes="$$skipped"; \
+	  fi; \
+	  report=""; \
+	  if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \
+	    report="Please report to $(PACKAGE_BUGREPORT)"; \
+	    test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \
+	      dashes="$$report"; \
+	  fi; \
+	  dashes=`echo "$$dashes" | sed s/./=/g`; \
+	  if test "$$failed" -eq 0; then \
+	    col="$$grn"; \
+	  else \
+	    col="$$red"; \
+	  fi; \
+	  echo "$${col}$$dashes$${std}"; \
+	  echo "$${col}$$banner$${std}"; \
+	  test -z "$$skipped" || echo "$${col}$$skipped$${std}"; \
+	  test -z "$$report" || echo "$${col}$$report$${std}"; \
+	  echo "$${col}$$dashes$${std}"; \
+	  test "$$failed" -eq 0; \
+	else :; fi
+check-am: all-am
+	$(MAKE) $(AM_MAKEFLAGS) $(check_PROGRAMS)
+	$(MAKE) $(AM_MAKEFLAGS) check-TESTS
+check: check-am
+all-am: Makefile $(LTLIBRARIES) all-multi config.h
+installdirs:
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am clean-multi
+
+clean-am: clean-checkPROGRAMS clean-generic clean-libtool \
+	clean-noinstLTLIBRARIES mostlyclean-am
+
+distclean: distclean-am distclean-multi
+	-rm -f $(am__CONFIG_DISTCLEAN_FILES)
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-hdr distclean-libtool distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am:
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am: install-multi
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am maintainer-clean-multi
+	-rm -f $(am__CONFIG_DISTCLEAN_FILES)
+	-rm -rf $(top_srcdir)/autom4te.cache
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am mostlyclean-multi
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am:
+
+.MAKE: all all-multi check-am clean-multi distclean-multi install-am \
+	install-multi install-strip maintainer-clean-multi \
+	mostlyclean-multi
+
+.PHONY: CTAGS GTAGS all all-am all-multi am--refresh check check-TESTS \
+	check-am clean clean-checkPROGRAMS clean-generic clean-libtool \
+	clean-multi clean-noinstLTLIBRARIES ctags distclean \
+	distclean-compile distclean-generic distclean-hdr \
+	distclean-libtool distclean-multi distclean-tags dvi dvi-am \
+	html html-am info info-am install install-am install-data \
+	install-data-am install-dvi install-dvi-am install-exec \
+	install-exec-am install-html install-html-am install-info \
+	install-info-am install-man install-multi install-pdf \
+	install-pdf-am install-ps install-ps-am install-strip \
+	installcheck installcheck-am installdirs maintainer-clean \
+	maintainer-clean-generic maintainer-clean-multi mostlyclean \
+	mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
+	mostlyclean-multi pdf pdf-am ps ps-am tags uninstall \
+	uninstall-am
+
+alloc.lo: config.h backtrace.h internal.h
+backtrace.lo: config.h backtrace.h internal.h
+btest.lo: (INCDIR)/filenames.h backtrace.h backtrace-supported.h
+dwarf.lo: config.h $(INCDIR)/dwarf2.h $(INCDIR)/dwarf2.def \
+	$(INCDIR)/filenames.h backtrace.h internal.h
+elf.lo: config.h backtrace.h internal.h
+fileline.lo: config.h backtrace.h internal.h
+mmap.lo: config.h backtrace.h internal.h
+mmapio.lo: config.h backtrace.h internal.h
+nounwind.lo: config.h internal.h
+pecoff.lo: config.h backtrace.h internal.h
+posix.lo: config.h backtrace.h internal.h
+print.lo: config.h backtrace.h internal.h
+read.lo: config.h backtrace.h internal.h
+simple.lo: config.h backtrace.h internal.h
+sort.lo: config.h backtrace.h internal.h
+stest.lo: config.h backtrace.h internal.h
+state.lo: config.h backtrace.h backtrace-supported.h internal.h
+unknown.lo: config.h backtrace.h internal.h
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/Makefile.am b/rpython/rlib/rvmprof/src/libbacktrace/Makefile.am
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/libbacktrace/Makefile.am
@@ -0,0 +1,136 @@
+# Makefile.am -- Backtrace Makefile.
+# Copyright (C) 2012-2017 Free Software Foundation, Inc.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+#     (1) Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+
+#     (2) Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+
+#     (3) The name of the author may not be used to
+#     endorse or promote products derived from this software without
+#     specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+ACLOCAL_AMFLAGS = -I .. -I ../config
+
+AM_CPPFLAGS = -I $(top_srcdir)/../include -I $(top_srcdir)/../libgcc \
+	-I ../libgcc
+
+AM_CFLAGS = $(EXTRA_FLAGS) $(WARN_FLAGS) $(PIC_FLAG)
+
+noinst_LTLIBRARIES = libbacktrace.la
+
+libbacktrace_la_SOURCES = \
+	backtrace.h \
+	atomic.c \
+	dwarf.c \
+	fileline.c \
+	internal.h \
+	posix.c \
+	print.c \
+	sort.c \
+	state.c
+
+BACKTRACE_FILES = \
+	backtrace.c \
+	simple.c \
+	nounwind.c
+
+FORMAT_FILES = \
+	elf.c \
+	pecoff.c \
+	unknown.c
+
+VIEW_FILES = \
+	read.c \
+	mmapio.c
+
+ALLOC_FILES = \
+	alloc.c \
+	mmap.c
+
+EXTRA_libbacktrace_la_SOURCES = \
+	$(BACKTRACE_FILES) \
+	$(FORMAT_FILES) \
+	$(VIEW_FILES) \
+	$(ALLOC_FILES)
+
+libbacktrace_la_LIBADD = \
+	$(BACKTRACE_FILE) \
+	$(FORMAT_FILE) \
+	$(VIEW_FILE) \
+	$(ALLOC_FILE)
+
+libbacktrace_la_DEPENDENCIES = $(libbacktrace_la_LIBADD)
+
+# Testsuite.
+
+check_PROGRAMS =
+
+TESTS = $(check_PROGRAMS)
+
+if NATIVE
+
+btest_SOURCES = btest.c
+btest_CFLAGS = $(AM_CFLAGS) -g -O
+btest_LDADD = libbacktrace.la
+
+check_PROGRAMS += btest
+
+stest_SOURCES = stest.c
+stest_LDADD = libbacktrace.la
+
+check_PROGRAMS += stest
+
+endif NATIVE
+
+# We can't use automake's automatic dependency tracking, because it
+# breaks when using bootstrap-lean.  Automatic dependency tracking
+# with GCC bootstrap will cause some of the objects to depend on
+# header files in prev-gcc/include, e.g., stddef.h and stdarg.h.  When
+# using bootstrap-lean, prev-gcc is removed after each stage.  When
+# running "make install", those header files will be gone, causing the
+# library to be rebuilt at install time.  That may not succeed.
+
+# These manual dependencies do not include dependencies on unwind.h,
+# even though that is part of GCC, because where to find it depends on
+# whether we are being built as a host library or a target library.
+
+INCDIR = $(top_srcdir)/../include
+alloc.lo: config.h backtrace.h internal.h
+backtrace.lo: config.h backtrace.h internal.h
+btest.lo: (INCDIR)/filenames.h backtrace.h backtrace-supported.h
+dwarf.lo: config.h $(INCDIR)/dwarf2.h $(INCDIR)/dwarf2.def \
+	$(INCDIR)/filenames.h backtrace.h internal.h
+elf.lo: config.h backtrace.h internal.h
+fileline.lo: config.h backtrace.h internal.h
+mmap.lo: config.h backtrace.h internal.h
+mmapio.lo: config.h backtrace.h internal.h
+nounwind.lo: config.h internal.h
+pecoff.lo: config.h backtrace.h internal.h
+posix.lo: config.h backtrace.h internal.h
+print.lo: config.h backtrace.h internal.h
+read.lo: config.h backtrace.h internal.h
+simple.lo: config.h backtrace.h internal.h
+sort.lo: config.h backtrace.h internal.h
+stest.lo: config.h backtrace.h internal.h
+state.lo: config.h backtrace.h backtrace-supported.h internal.h
+unknown.lo: config.h backtrace.h internal.h
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/Makefile.in b/rpython/rlib/rvmprof/src/libbacktrace/Makefile.in
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/libbacktrace/Makefile.in
@@ -0,0 +1,770 @@
+# Makefile.in generated by automake 1.11.6 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+# Foundation, Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+ at SET_MAKE@
+
+# Makefile.am -- Backtrace Makefile.
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+#     (1) Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+
+#     (2) Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+
+#     (3) The name of the author may not be used to
+#     endorse or promote products derived from this software without
+#     specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+VPATH = @srcdir@
+am__make_dryrun = \
+  { \
+    am__dry=no; \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        echo 'am--echo: ; @echo "AM"  OK' | $(MAKE) -f - 2>/dev/null \
+          | grep '^AM OK$$' >/dev/null || am__dry=yes;; \
+      *) \
+        for am__flg in $$MAKEFLAGS; do \
+          case $$am__flg in \
+            *=*|--*) ;; \
+            *n*) am__dry=yes; break;; \
+          esac; \
+        done;; \
+    esac; \
+    test $$am__dry = yes; \
+  }
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+target_triplet = @target@
+check_PROGRAMS = $(am__EXEEXT_1)
+ at NATIVE_TRUE@am__append_1 = btest stest
+subdir = .
+DIST_COMMON = README ChangeLog $(srcdir)/Makefile.in \
+	$(srcdir)/Makefile.am $(top_srcdir)/configure \
+	$(am__configure_deps) $(srcdir)/config.h.in \
+	$(srcdir)/../mkinstalldirs $(srcdir)/backtrace-supported.h.in
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/../config/lead-dot.m4 \
+	$(top_srcdir)/../config/multi.m4 \
+	$(top_srcdir)/../config/override.m4 \
+	$(top_srcdir)/../config/stdint.m4 \
+	$(top_srcdir)/../config/unwind_ipinfo.m4 \
+	$(top_srcdir)/../config/warnings.m4 \
+	$(top_srcdir)/../libtool.m4 $(top_srcdir)/../ltoptions.m4 \
+	$(top_srcdir)/../ltsugar.m4 $(top_srcdir)/../ltversion.m4 \
+	$(top_srcdir)/../lt~obsolete.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
+ configure.lineno config.status.lineno
+mkinstalldirs = $(SHELL) $(top_srcdir)/../mkinstalldirs
+CONFIG_HEADER = config.h
+CONFIG_CLEAN_FILES = backtrace-supported.h
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+am__DEPENDENCIES_1 =
+am_libbacktrace_la_OBJECTS = atomic.lo dwarf.lo fileline.lo posix.lo \
+	print.lo sort.lo state.lo
+libbacktrace_la_OBJECTS = $(am_libbacktrace_la_OBJECTS)
+ at NATIVE_TRUE@am__EXEEXT_1 = btest$(EXEEXT) stest$(EXEEXT)
+ at NATIVE_TRUE@am_btest_OBJECTS = btest-btest.$(OBJEXT)
+btest_OBJECTS = $(am_btest_OBJECTS)
+ at NATIVE_TRUE@btest_DEPENDENCIES = libbacktrace.la
+btest_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=link $(CCLD) $(btest_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
+	$(LDFLAGS) -o $@
+ at NATIVE_TRUE@am_stest_OBJECTS = stest.$(OBJEXT)
+stest_OBJECTS = $(am_stest_OBJECTS)
+ at NATIVE_TRUE@stest_DEPENDENCIES = libbacktrace.la
+DEFAULT_INCLUDES = -I. at am__isrc@
+depcomp =
+am__depfiles_maybe =
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+CCLD = $(CC)
+LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
+	$(LDFLAGS) -o $@
+SOURCES = $(libbacktrace_la_SOURCES) $(EXTRA_libbacktrace_la_SOURCES) \
+	$(btest_SOURCES) $(stest_SOURCES)
+MULTISRCTOP = 
+MULTIBUILDTOP = 
+MULTIDIRS = 
+MULTISUBDIR = 
+MULTIDO = true
+MULTICLEAN = true
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+ETAGS = etags
+CTAGS = ctags
+am__tty_colors = \
+red=; grn=; lgn=; blu=; std=
+ACLOCAL = @ACLOCAL@
+ALLOC_FILE = @ALLOC_FILE@
+AMTAR = @AMTAR@

From pypy.commits at gmail.com  Mon Feb  6 10:15:06 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 06 Feb 2017 07:15:06 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: moved shared vmprof files into
 folder shared/, working on reorganization
Message-ID: <589892fa.5190df0a.265de.aa2c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89974:39d94987a416
Date: 2017-02-06 16:11 +0100
http://bitbucket.org/pypy/pypy/changeset/39d94987a416/

Log:	moved shared vmprof files into folder shared/, working on
	reorganization

diff too long, truncating to 2000 out of 52275 lines

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -14,11 +14,13 @@
 
 ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof')
 SRC = ROOT.join('src')
+SHARED = SRC.join('shared')
 UDIS86 = ROOT.join('libudis86')
 BACKTRACE = ROOT.join('libbacktrace')
 
+compile_extra = ['-DRPYTHON_LL2CTYPES','-DRPYTHON_VMPROF']
 if sys.platform.startswith('linux'):
-    extra_compile = [
+    separate_module_files = [
        BACKTRACE.join('backtrace.c'),
        BACKTRACE.join('state.c'),
        BACKTRACE.join('elf.c'),
@@ -30,38 +32,44 @@
        BACKTRACE.join('sort.c'),
     ]
     _libs = ['dl']
+    compile_extra += ['-DVMPROF_UNIX']
+elif sys.platform == 'darwin':
+    compile_extra += ['-DVMPROF_UNIX']
+    separate_module_files = []
+    _libs = []
 else:
-    extra_compile = []
+    # windows
+    separate_module_files = []
     _libs = []
+
 eci_kwds = dict(
-    include_dirs = [SRC],
+    include_dirs = [SRC, SHARED, BACKTRACE, UDIS86],
     includes = ['rvmprof.h', 'vmprof_stack.h'],
     libraries = _libs,
     separate_module_files = [
         SRC.join('rvmprof.c'),
-        SRC.join('compat.c'),
-        SRC.join('machine.c'),
-        SRC.join('symboltable.c'),
-        SRC.join('stack.c'),
+        SHARED.join('compat.c'),
+        SHARED.join('machine.c'),
+        SHARED.join('symboltable.c'),
+        SHARED.join('stack.c'),
         # udis86
-        SRC.join('libudis86/decode.c'),
-        SRC.join('libudis86/itab.c'),
-        SRC.join('libudis86/udis86.c'),
-    ] + extra_compile,
-    post_include_bits=['#define RPYTHON_VMPROF\n'],
+        SHARED.join('libudis86/decode.c'),
+        SHARED.join('libudis86/itab.c'),
+        SHARED.join('libudis86/udis86.c'),
+    ] + separate_module_files,
+    post_include_bits=[],
+    compile_extra=compile_extra
     )
 global_eci = ExternalCompilationInfo(**eci_kwds)
 
 
 def setup():
-    compile_extra = ['-DRPYTHON_LL2CTYPES']
-    platform.verify_eci(ExternalCompilationInfo(
-        compile_extra=compile_extra,
-        **eci_kwds))
+    platform.verify_eci(ExternalCompilationInfo(**eci_kwds))
 
     eci = global_eci
     vmprof_init = rffi.llexternal("vmprof_init",
-                                  [rffi.INT, rffi.DOUBLE, rffi.CCHARP],
+                                  [rffi.INT, rffi.DOUBLE, rffi.INT, rffi.INT,
+                                   rffi.CCHARP, rffi.INT],
                                   rffi.CCHARP, compilation_info=eci)
     vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
                                     compilation_info=eci,
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog b/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog
+++ /dev/null
@@ -1,602 +0,0 @@
-2017-01-01  Jakub Jelinek  <jakub at redhat.com>
-
-	Update copyright years.
-
-2016-11-15  Matthias Klose  <doko at ubuntu.com>
-
-	* configure: Regenerate.
-
-2016-09-11  Carlos Liam  <carlos at aarzee.me>
-
-	* all: Remove meaningless trailing whitespace.
-
-2016-05-18  Uros Bizjak  <ubizjak at gmail.com>
-
-	PR target/71161
-	* elf.c (phdr_callback) [__i386__]: Add
-	__attribute__((__force_align_arg_pointer__)).
-
-2016-03-02  Maxim Ostapenko  <m.ostapenko at partner.samsung.com>
-
-	* elf.c (backtrace_initialize): Properly initialize elf_fileline_fn to
-	avoid possible crash.
-	(elf_add): Don't set *fileline_fn to elf_nodebug value in case of
-	missing debug info anymore.
-
-2016-02-06  John David Anglin  <danglin at gcc.gnu.org>
-
-	* mmap.c (MAP_FAILED): Define if not defined.
-
-2016-01-04  Jakub Jelinek  <jakub at redhat.com>
-
-	Update copyright years.
-
-2015-12-18  Andris Pavenis  <andris.pavenis at iki.fi>
-
-	* configure.ac: Specify that DJGPP do not have mmap
-	even when sys/mman.h exists.
-	* configure: Regenerate
-
-2015-12-09  John David Anglin  <danglin at gcc.gnu.org>
-
-	PR libgfortran/68115
-	* configure.ac: Set libbacktrace_cv_sys_sync to no on hppa*-*-hpux*.
-	* configure: Regenerate.
-	* elf.c (backtrace_initialize): Cast __sync_bool_compare_and_swap call
-	to void.
-
-2015-09-17  Ian Lance Taylor  <iant at google.com>
-
-	* posix.c (backtrace_open): Cast second argument of open() to int.
-
-2015-09-11  Ian Lance Taylor  <iant at google.com>
-
-	* Makefile.am (backtrace.lo): Depend on internal.h.
-	(sort.lo, stest.lo): Add explicit dependencies.
-	* Makefile.in: Rebuild.
-
-2015-09-09  Hans-Peter Nilsson  <hp at axis.com>
-
-	* backtrace.c: #include <sys/types.h>.
-
-2015-09-08  Ian Lance Taylor  <iant at google.com>
-
-	PR other/67457
-	* backtrace.c: #include "internal.h".
-	(struct backtrace_data): Add can_alloc field.
-	(unwind): If can_alloc is false, don't try to get file/line
-	information.
-	(backtrace_full): Set can_alloc field in bdata.
-	* alloc.c (backtrace_alloc): Don't call error_callback if it is
-	NULL.
-	* mmap.c (backtrace_alloc): Likewise.
-	* internal.h: Update comments for backtrace_alloc and
-	backtrace_free.
-
-2015-09-08  Ian Lance Taylor  <iant at google.com>
-
-	PR other/67457
-	* mmap.c (backtrace_alloc): Correct test for mmap failure.
-
-2015-08-31  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
-
-	* configure.ac: For spu-*-* targets, set have_fcntl to no.
-	* configure: Regenerate.
-
-2015-08-27  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
-
-	* configure.ac: Remove [disable-shared] argument to LT_INIT.
-	Remove setting PIC_FLAG when building as target library.
-	* configure: Regenerate.
-
-2015-08-26  Hans-Peter Nilsson  <hp at axis.com>
-
-	* configure.ac: Only compile with -fPIC if the target
-	supports it.
-	* configure: Regenerate.
-
-2015-08-24  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
-
-	* configure.ac: Set have_mmap to no on spu-*-* targets.
-	* configure: Regenerate.
-
-2015-08-13  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_function_entry): Add vec_inlined parameter.
-	Change all callers.
-
-2015-06-11  Martin Sebor  <msebor at redhat.com>
-
-	PR sanitizer/65479
-	* dwarf.c (struct line): Add new field idx.
-	(line_compare): Use it.
-	(add_line): Set it.
-	(read_line_info): Reset it.
-
-2015-05-29  Tristan Gingold  <gingold at adacore.com>
-
-	* pecoff.c: New file.
-	* Makefile.am (FORMAT_FILES): Add pecoff.c and dependencies.
-	* Makefile.in: Regenerate.
-	* filetype.awk: Detect pecoff.
-	* configure.ac: Define BACKTRACE_SUPPORTS_DATA on elf platforms.
-	Add pecoff.
-	* btest.c (test5): Test enabled only if BACKTRACE_SUPPORTS_DATA is
-	true.
-	* backtrace-supported.h.in (BACKTRACE_SUPPORTS_DATA): Define.
-	* configure: Regenerate.
-	* pecoff.c: New file.
-
-2015-05-13  Michael Haubenwallner  <michael.haubenwallner at ssi-schaefer.com>
-
-	* Makefile.in: Regenerated with automake-1.11.6.
-	* aclocal.m4: Likewise.
-	* configure: Likewise.
-
-2015-01-24  Matthias Klose  <doko at ubuntu.com>
-
-	* configure.ac: Move AM_ENABLE_MULTILIB before AC_PROG_CC.
-	* configure: Regenerate.
-
-2015-01-05  Jakub Jelinek  <jakub at redhat.com>
-
-	Update copyright years.
-
-2014-11-21  H.J. Lu  <hongjiu.lu at intel.com>
-
-	PR bootstrap/63784
-	* configure: Regenerated.
-
-2014-11-11  David Malcolm  <dmalcolm at redhat.com>
-
-	* ChangeLog.jit: New.
-
-2014-11-11  Francois-Xavier Coudert  <fxcoudert at gcc.gnu.org>
-
-	PR target/63610
-	* configure: Regenerate.
-
-2014-10-23  Ian Lance Taylor  <iant at google.com>
-
-	* internal.h (backtrace_atomic_load_pointer) [no atomic or sync]:
-	Fix to return void *.
-
-2014-05-08  Ian Lance Taylor  <iant at google.com>
-
-	* mmap.c (backtrace_free): If freeing a large aligned block of
-	memory, call munmap rather than holding onto it.
-	(backtrace_vector_grow): When growing a vector, double the number
-	of pages requested.  When releasing the old version of a grown
-	vector, pass the correct size to backtrace_free.
-
-2014-03-07  Ian Lance Taylor  <iant at google.com>
-
-	* sort.c (backtrace_qsort): Use middle element as pivot.
-
-2014-03-06  Ian Lance Taylor  <iant at google.com>
-
-	* sort.c: New file.
-	* stest.c: New file.
-	* internal.h (backtrace_qsort): Declare.
-	* dwarf.c (read_abbrevs): Call backtrace_qsort instead of qsort.
-	(read_line_info, read_function_entry): Likewise.
-	(read_function_info, build_dwarf_data): Likewise.
-	* elf.c (elf_initialize_syminfo): Likewise.
-	* Makefile.am (libbacktrace_la_SOURCES): Add sort.c.
-	(stest_SOURCES, stest_LDADD): Define.
-	(check_PROGRAMS): Add stest.
-
-2014-02-07  Misty De Meo  <misty at brew.sh>
-
-	PR target/58710
-	* configure.ac: Use AC_LINK_IFELSE in check for
-	_Unwind_GetIPInfo.
-	* configure: Regenerate.
-
-2014-01-02  Richard Sandiford  <rdsandiford at googlemail.com>
-
-	Update copyright years
-
-2013-12-06  Jakub Jelinek  <jakub at redhat.com>
-
-	* elf.c (ET_DYN): Undefine and define again.
-	(elf_add): Add exe argument, if true and ehdr.e_type is ET_DYN,
-	return early -1 without closing the descriptor.
-	(struct phdr_data): Add exe_descriptor.
-	(phdr_callback): If pd->exe_descriptor is not -1, for very first
-	call if dlpi_name is NULL just call elf_add with the exe_descriptor,
-	otherwise backtrace_close the exe_descriptor if not -1.  Adjust
-	call to elf_add.
-	(backtrace_initialize): Adjust call to elf_add.  If it returns
-	-1, set pd.exe_descriptor to descriptor, otherwise set it to -1.
-
-2013-12-05  Ian Lance Taylor  <iant at google.com>
-
-	* alloc.c (backtrace_vector_finish): Add error_callback and data
-	parameters.  Call backtrace_vector_release.  Return address base.
-	* mmap.c (backtrace_vector_finish): Add error_callback and data
-	parameters.  Return address base.
-	* dwarf.c (read_function_info): Get new address base from
-	backtrace_vector_finish.
-	* internal.h (backtrace_vector_finish): Update declaration.
-
-2013-11-27  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (find_address_ranges): New static function, broken out
-	of build_address_map.
-	(build_address_map): Call it.
-	* btest.c (check): Check for missing filename or function, rather
-	than crashing.
-	(f3): Check that enough frames were returned.
-
-2013-11-19  Jakub Jelinek  <jakub at redhat.com>
-
-	* backtrace.h (backtrace_syminfo_callback): Add symsize argument.
-	* elf.c (elf_syminfo): Pass 0 or sym->size to the callback as
-	last argument.
-	* btest.c (struct symdata): Add size field.
-	(callback_three): Add symsize argument.  Copy it to the data->size
-	field.
-	(f23): Set symdata.size to 0.
-	(test5): Likewise.  If sizeof (int) > 1, lookup address of
-	((uintptr_t) &global) + 1.  Verify symdata.val and symdata.size
-	values.
-
-	* atomic.c: Include sys/types.h.
-
-2013-11-18  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Check for support of __atomic extensions.
-	* internal.h: Declare or #define atomic functions for use in
-	backtrace code.
-	* atomic.c: New file.
-	* dwarf.c (dwarf_lookup_pc): Use atomic functions.
-	(dwarf_fileline, backtrace_dwarf_add): Likewise.
-	* elf.c (elf_add_syminfo_data, elf_syminfo): Likewise.
-	(backtrace_initialize): Likewise.
-	* fileline.c (fileline_initialize): Likewise.
-	* Makefile.am (libbacktrace_la_SOURCES): Add atomic.c.
-	* configure, config.h.in, Makefile.in: Rebuild.
-
-2013-11-18  Jakub Jelinek  <jakub at redhat.com>
-
-	* elf.c (SHN_UNDEF): Define.
-	(elf_initialize_syminfo): Add base_address argument.  Ignore symbols
-	with st_shndx == SHN_UNDEF.  Add base_address to address fields.
-	(elf_add): Adjust caller.
-
-	* elf.c (phdr_callback): Process info->dlpi_addr == 0 normally.
-
-2013-11-16  Ian Lance Taylor  <iant at google.com>
-
-	* backtrace.h (backtrace_create_state): Correct comment about
-	threading.
-
-2013-11-15  Ian Lance Taylor  <iant at google.com>
-
-	* backtrace.h (backtrace_syminfo): Update comment and parameter
-	name to take any address, not just a PC value.
-	* elf.c (STT_OBJECT): Define.
-	(elf_nosyms): Rename parameter pc to addr.
-	(elf_symbol_search): Rename local variable pc to addr.
-	(elf_initialize_syminfo): Add STT_OBJECT symbols to elf_symbols.
-	(elf_syminfo): Rename parameter pc to addr.
-	* btest.c (global): New global variable.
-	(test5): New test.
-	(main): Call test5.
-
-2013-10-17  Ian Lance Taylor  <iant at google.com>
-
-	* elf.c (elf_add): Don't get the wrong offsets if a debug section
-	is missing.
-
-2013-10-15  David Malcolm  <dmalcolm at redhat.com>
-
-	* configure.ac: Add --enable-host-shared, setting up
-	pre-existing PIC_FLAG variable within Makefile.am et al.
-	* configure: Regenerate.
-
-2013-09-20  Alan Modra  <amodra at gmail.com>
-
-	* configure: Regenerate.
-
-2013-07-23  Alexander Monakov  <amonakov at ispras.ru>
-
-	* elf.c (elf_syminfo): Loop over the elf_syminfo_data chain.
-
-2013-07-23  Alexander Monakov  <amonakov at ispras.ru>
-
-	* elf.c (backtrace_initialize): Pass elf_fileline_fn to
-	dl_iterate_phdr callbacks.
-
-2013-03-25  Ian Lance Taylor  <iant at google.com>
-
-	* alloc.c: #include <sys/types.h>.
-	* mmap.c: Likewise.
-
-2013-01-31  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_function_info): Permit fvec parameter to be NULL.
-	(dwarf_lookup_pc): Don't use ddata->fvec if threaded.
-
-2013-01-25  Jakub Jelinek  <jakub at redhat.com>
-
-	PR other/56076
-	* dwarf.c (read_line_header): Don't crash if DW_AT_comp_dir
-	attribute was not seen.
-
-2013-01-16  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (struct unit): Add filename and abs_filename fields.
-	(build_address_map): Set new fields when reading unit.
-	(dwarf_lookup_pc): If we don't find an entry in the line table,
-	just return the main file name.
-
-2013-01-14  Richard Sandiford  <rdsandiford at googlemail.com>
-
-	Update copyright years.
-
-2013-01-01  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54834
-	* Makefile.am (AM_CPPFLAGS): Remove -I ../gcc/include and -I
-	$(MULTIBUILDTOP)/../../gcc/include.
-	* Makefile.in: Rebuild.
-
-2013-01-01  Ian Lance Taylor  <iant at google.com>
-
-	PR other/55536
-	* mmap.c (backtrace_alloc): Don't call sync functions if not
-	threaded.
-	(backtrace_free): Likewise.
-
-2012-12-12  John David Anglin  <dave.anglin at nrc-cnrc.gc.ca>
-
-	* mmapio.c: Define MAP_FAILED if not defined.
-
-2012-12-11  Jakub Jelinek  <jakub at redhat.com>
-
-	PR bootstrap/54926
-	* Makefile.am (AM_CFLAGS): Remove -frandom-seed=$@.
-	* configure.ac: If --with-target-subdir, add -frandom-seed=$@
-	to EXTRA_FLAGS unconditionally, otherwise check whether the compiler
-	accepts it.
-	* Makefile.in: Regenerated.
-	* configure: Regenerated.
-
-2012-12-07  Jakub Jelinek  <jakub at redhat.com>
-
-	PR bootstrap/54926
-	* Makefile.am (AM_CFLAGS): Add -frandom-seed=$@.
-	* Makefile.in: Regenerated.
-
-2012-11-20  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_attribute): Always clear val.
-
-2012-11-13  Ian Lance Taylor  <iant at google.com>
-
-	PR other/55312
-	* configure.ac: Only add -Werror if building a target library.
-	* configure: Rebuild.
-
-2012-11-12  Ian Lance Taylor  <iant at google.com>
-	    Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
-	    Gerald Pfeifer  <gerald at pfeifer.com>
-
-	* configure.ac: Check for getexecname.
-	* fileline.c: #include <errno.h>.  Define getexecname if not
-	available.
-	(fileline_initialize): Try to find the executable in a few
-	different ways.
-	* print.c (error_callback): Only print the filename if it came
-	from the backtrace state.
-	* configure, config.h.in: Rebuild.
-
-2012-10-29  Ian Lance Taylor  <iant at google.com>
-
-	* mmap.c (backtrace_vector_release): Correct last patch: add
-	aligned, not size.
-
-2012-10-29  Ian Lance Taylor  <iant at google.com>
-
-	* mmap.c (backtrace_vector_release): Make sure freed block is
-	aligned on 8-byte boundary.
-
-2012-10-26  Ian Lance Taylor  <iant at google.com>
-
-	PR other/55087
-	* posix.c (backtrace_open): Add does_not_exist parameter.
-	* elf.c (phdr_callback): Do not warn if shared library could not
-	be opened.
-	* fileline.c (fileline_initialize): Update calls to
-	backtrace_open.
-	* internal.h (backtrace_open): Update declaration.
-
-2012-10-26  Jack Howarth  <howarth at bromo.med.uc.edu>
-
-	PR target/55061
-	* configure.ac: Check for _Unwind_GetIPInfo function declaration.
-	* configure: Regenerate.
-
-2012-10-24  Ian Lance Taylor  <iant at google.com>
-
-	PR target/55061
-	* configure.ac: Check whether -funwind-tables option works.
-	* configure: Rebuild.
-
-2012-10-11  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Do not use dl_iterate_phdr on Solaris 10.
-	* configure: Rebuild.
-
-2012-10-10  Ian Lance Taylor  <iant at google.com>
-
-	* elf.c: Rename all Elf typedefs to start with b_elf, and be all
-	lower case.
-
-2012-10-10  Hans-Peter Nilsson  <hp at bitrange.com>
-
-	* elf.c (elf_add_syminfo_data): Add casts to avoid warning.
-
-2012-10-09  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (dwarf_fileline): Add cast to avoid warning.
-	(backtrace_dwarf_add): Likewise.
-
-2012-10-09  Ian Lance Taylor  <iant at google.com>
-
-	Add support for tracing through shared libraries.
-	* configure.ac: Check for link.h and dl_iterate_phdr.
-	* elf.c: #include <link.h> if system has dl_iterate_phdr.  #undef
-	ELF macros before #defining them.
-	(dl_phdr_info, dl_iterate_phdr): Define if system does not have
-	dl_iterate_phdr.
-	(struct elf_syminfo_data): Add next field.
-	(elf_initialize_syminfo): Initialize next field.
-	(elf_add_syminfo_data): New static function.
-	(elf_add): New static function, broken out of
-	backtrace_initialize.  Call backtrace_dwarf_add instead of
-	backtrace_dwarf_initialize.
-	(struct phdr_data): Define.
-	(phdr_callback): New static function.
-	(backtrace_initialize): Call elf_add.
-	* dwarf.c (struct dwarf_data): Add next and base_address fields.
-	(add_unit_addr): Add base_address parameter.  Change all callers.
-	(add_unit_ranges, build_address_map): Likewise.
-	(add_line): Add ddata parameter.  Change all callers.
-	(read_line_program, add_function_range): Likewise.
-	(dwarf_lookup_pc): New static function, broken out of
-	dwarf_fileline.
-	(dwarf_fileline): Call dwarf_lookup_pc.
-	(build_dwarf_data): New static function.
-	(backtrace_dwarf_add): New function.
-	(backtrace_dwarf_initialize): Remove.
-	* internal.h (backtrace_dwarf_initialize): Don't declare.
-	(backtrace_dwarf_add): Declare.
-	* configure, config.h.in: Rebuild.
-
-2012-10-04  Gerald Pfeifer  <gerald at pfeifer.com>
-
-	* btest.c (f23): Avoid uninitialized variable warning.
-
-2012-10-04  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c: If the system header files do not declare strnlen,
-	provide our own version.
-
-2012-10-03  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_uleb128): Fix overflow test.
-	(read_sleb128): Likewise.
-	(build_address_map): Don't change unit_buf.start.
-
-2012-10-02  Uros Bizjak  <ubizjak at gmail.com>
-
-	PR other/54761
-	* configure.ac (EXTRA_FLAGS): New.
-	* Makefile.am (AM_FLAGS): Add $(EXTRA_FLAGS).
-	* configure, Makefile.in: Regenerate.
-
-2012-09-29  Ian Lance Taylor  <iant at google.com>
-
-	PR other/54749
-	* fileline.c (fileline_initialize): Pass errnum as -1 when
-	reporting that we could not read executable information after a
-	previous failure.
-
-2012-09-27  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54732
-	* configure.ac: Add no-dependencies to AM_INIT_AUTOMAKE.
-	* Makefile.am: Add dependencies for all objects.
-	* configure, aclocal.m4, Makefile.in: Rebuild.
-
-2012-09-27  Ian Lance Taylor  <iant at google.com>
-
-	PR other/54726
-	* elf.c (backtrace_initialize): Set *fileln_fn, not
-	state->fileln_fn.
-
-2012-09-19  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Only use GCC_CHECK_UNWIND_GETIPINFO when compiled
-	as a target library.
-	* configure: Rebuild.
-
-2012-09-19  Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
-	    Ian Lance Taylor  <iant at google.com>
-
-        * configure.ac (GCC_HEADER_STDINT): Invoke.
-        * backtrace.h: If we can't find <stdint.h>, use "gstdint.h".
-        * btest.c: Don't include <stdint.h>.
-        * dwarf.c: Likewise.
-        * configure, aclocal.m4, Makefile.in, config.h.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54623
-	* Makefile.am (AM_CPPFLAGS): Define.
-	(AM_CFLAGS): Remove -I options.
-	* Makefile.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* posix.c (O_BINARY): Define if not defined.
-	(backtrace_open): Pass O_BINARY to open.  Only call fcntl if
-	HAVE_FCNTL is defined.
-	* configure.ac: Test for the fcntl function.
-	* configure, config.h.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* btest.c (test1, test2, test3, test4): Add the unused attribute.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c: Correct test of HAVE_DECL_STRNLEN.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Add AC_USE_SYSTEM_EXTENSIONS.
-	* mmapio.c: Don't define _GNU_SOURCE.
-	* configure, config.h.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Check whether strnlen is declared.
-	* dwarf.c: Declare strnlen if not declared.
-	* configure, config.h.in: Rebuild.
-
-2012-09-18  Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
-
-	* fileline.c: Include <stdlib.h>.
-	* mmap.c: Likewise.
-
-2012-09-17  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54611
-	* nounwind.c (backtrace_full): Rename from backtrace.  Add state
-	parameter.
-
-2012-09-17  Gerald Pfeifer  <gerald at pfeifer.com>
-
-	PR bootstrap/54611
-	* nounwind.c (backtrace_simple): Add state parameter.
-
-2012-09-17  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54609
-	* unknown.c (unknown_fileline): Add state parameter, remove
-	fileline_data parameter, name error_callback parameter.
-	(backtrace_initialize): Add state parameter.
-
-2012-09-17  Ian Lance Taylor  <iant at google.com>
-
-	* Initial implementation.
-

-Copyright (C) 2012-2017 Free Software Foundation, Inc.
-
-Copying and distribution of this file, with or without modification,
-are permitted in any medium without royalty provided the copyright
-notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog.jit b/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog.jit
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/libbacktrace/ChangeLog.jit
+++ /dev/null
@@ -1,14 +0,0 @@
-2014-09-24  David Malcolm  <dmalcolm at redhat.com>
-
-	* ChangeLog.jit: Add copyright footer.
-
-2013-10-03  David Malcolm  <dmalcolm at redhat.com>
-
-	* configure.ac: Add --enable-host-shared.
-	* configure: Regenerate.
-

-Copyright (C) 2013-2014 Free Software Foundation, Inc.
-
-Copying and distribution of this file, with or without modification,
-are permitted in any medium without royalty provided the copyright
-notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/Makefile b/rpython/rlib/rvmprof/src/libbacktrace/Makefile
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/libbacktrace/Makefile
+++ /dev/null
@@ -1,770 +0,0 @@
-# Makefile.in generated by automake 1.11.6 from Makefile.am.
-# Makefile.  Generated from Makefile.in by configure.
-
-# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
-# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-# Foundation, Inc.
-# This Makefile.in is free software; the Free Software Foundation
-# gives unlimited permission to copy and/or distribute it,
-# with or without modifications, as long as this notice is preserved.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
-# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE.
-
-
-
-# Makefile.am -- Backtrace Makefile.
-# Copyright (C) 2012-2016 Free Software Foundation, Inc.
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-
-#     (1) Redistributions of source code must retain the above copyright
-#     notice, this list of conditions and the following disclaimer.
-
-#     (2) Redistributions in binary form must reproduce the above copyright
-#     notice, this list of conditions and the following disclaimer in
-#     the documentation and/or other materials provided with the
-#     distribution.
-
-#     (3) The name of the author may not be used to
-#     endorse or promote products derived from this software without
-#     specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-
-am__make_dryrun = \
-  { \
-    am__dry=no; \
-    case $$MAKEFLAGS in \
-      *\\[\ \	]*) \
-        echo 'am--echo: ; @echo "AM"  OK' | $(MAKE) -f - 2>/dev/null \
-          | grep '^AM OK$$' >/dev/null || am__dry=yes;; \
-      *) \
-        for am__flg in $$MAKEFLAGS; do \
-          case $$am__flg in \
-            *=*|--*) ;; \
-            *n*) am__dry=yes; break;; \
-          esac; \
-        done;; \
-    esac; \
-    test $$am__dry = yes; \
-  }
-pkgdatadir = $(datadir)/libbacktrace
-pkgincludedir = $(includedir)/libbacktrace
-pkglibdir = $(libdir)/libbacktrace
-pkglibexecdir = $(libexecdir)/libbacktrace
-am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
-install_sh_DATA = $(install_sh) -c -m 644
-install_sh_PROGRAM = $(install_sh) -c
-install_sh_SCRIPT = $(install_sh) -c
-INSTALL_HEADER = $(INSTALL_DATA)
-transform = $(program_transform_name)
-NORMAL_INSTALL = :
-PRE_INSTALL = :
-POST_INSTALL = :
-NORMAL_UNINSTALL = :
-PRE_UNINSTALL = :
-POST_UNINSTALL = :
-build_triplet = x86_64-pc-linux-gnu
-host_triplet = x86_64-pc-linux-gnu
-target_triplet = x86_64-pc-linux-gnu
-check_PROGRAMS = $(am__EXEEXT_1)
-am__append_1 = btest stest
-subdir = .
-DIST_COMMON = README ChangeLog $(srcdir)/Makefile.in \
-	$(srcdir)/Makefile.am $(top_srcdir)/configure \
-	$(am__configure_deps) $(srcdir)/config.h.in \
-	$(srcdir)/../mkinstalldirs $(srcdir)/backtrace-supported.h.in
-ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/../config/lead-dot.m4 \
-	$(top_srcdir)/../config/multi.m4 \
-	$(top_srcdir)/../config/override.m4 \
-	$(top_srcdir)/../config/stdint.m4 \
-	$(top_srcdir)/../config/unwind_ipinfo.m4 \
-	$(top_srcdir)/../config/warnings.m4 \
-	$(top_srcdir)/../libtool.m4 $(top_srcdir)/../ltoptions.m4 \
-	$(top_srcdir)/../ltsugar.m4 $(top_srcdir)/../ltversion.m4 \
-	$(top_srcdir)/../lt~obsolete.m4 $(top_srcdir)/configure.ac
-am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
-	$(ACLOCAL_M4)
-am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
- configure.lineno config.status.lineno
-mkinstalldirs = $(SHELL) $(top_srcdir)/../mkinstalldirs
-CONFIG_HEADER = config.h
-CONFIG_CLEAN_FILES = backtrace-supported.h
-CONFIG_CLEAN_VPATH_FILES =
-LTLIBRARIES = $(noinst_LTLIBRARIES)
-am__DEPENDENCIES_1 =
-am_libbacktrace_la_OBJECTS = atomic.lo dwarf.lo fileline.lo posix.lo \
-	print.lo sort.lo state.lo
-libbacktrace_la_OBJECTS = $(am_libbacktrace_la_OBJECTS)
-am__EXEEXT_1 = btest$(EXEEXT) stest$(EXEEXT)
-am_btest_OBJECTS = btest-btest.$(OBJEXT)
-btest_OBJECTS = $(am_btest_OBJECTS)
-btest_DEPENDENCIES = libbacktrace.la
-btest_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=link $(CCLD) $(btest_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
-	$(LDFLAGS) -o $@
-am_stest_OBJECTS = stest.$(OBJEXT)
-stest_OBJECTS = $(am_stest_OBJECTS)
-stest_DEPENDENCIES = libbacktrace.la
-DEFAULT_INCLUDES = -I.
-depcomp =
-am__depfiles_maybe =
-COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
-	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
-LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
-	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
-CCLD = $(CC)
-LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
-	$(LDFLAGS) -o $@
-SOURCES = $(libbacktrace_la_SOURCES) $(EXTRA_libbacktrace_la_SOURCES) \
-	$(btest_SOURCES) $(stest_SOURCES)
-MULTISRCTOP = 
-MULTIBUILDTOP = 
-MULTIDIRS = 
-MULTISUBDIR = 
-MULTIDO = true
-MULTICLEAN = true
-am__can_run_installinfo = \
-  case $$AM_UPDATE_INFO_DIR in \
-    n|no|NO) false;; \
-    *) (install-info --version) >/dev/null 2>&1;; \
-  esac
-ETAGS = etags
-CTAGS = ctags
-am__tty_colors = \
-red=; grn=; lgn=; blu=; std=
-ACLOCAL = ${SHELL} /home/rich/src/gcc/missing --run aclocal-1.11
-ALLOC_FILE = mmap.lo
-AMTAR = $${TAR-tar}
-AR = ar
-AUTOCONF = ${SHELL} /home/rich/src/gcc/missing --run autoconf
-AUTOHEADER = ${SHELL} /home/rich/src/gcc/missing --run autoheader
-AUTOMAKE = ${SHELL} /home/rich/src/gcc/missing --run automake-1.11
-AWK = gawk
-BACKTRACE_FILE = backtrace.lo simple.lo
-BACKTRACE_SUPPORTED = 1
-BACKTRACE_SUPPORTS_DATA = 1
-BACKTRACE_SUPPORTS_THREADS = 1
-BACKTRACE_USES_MALLOC = 0
-CC = gcc
-CFLAGS = -g -O2
-CPP = gcc -E
-CPPFLAGS = 
-CYGPATH_W = echo
-DEFS = -DHAVE_CONFIG_H
-DSYMUTIL = 
-DUMPBIN = 
-ECHO_C = 
-ECHO_N = -n
-ECHO_T = 
-EGREP = /usr/bin/grep -E
-EXEEXT = 
-EXTRA_FLAGS = -funwind-tables -frandom-seed=$@
-FGREP = /usr/bin/grep -F
-FORMAT_FILE = elf.lo
-GREP = /usr/bin/grep
-INSTALL = /usr/bin/install -c
-INSTALL_DATA = ${INSTALL} -m 644
-INSTALL_PROGRAM = ${INSTALL}
-INSTALL_SCRIPT = ${INSTALL}
-INSTALL_STRIP_PROGRAM = $(install_sh) -c -s
-LD = /usr/bin/ld -m elf_x86_64
-LDFLAGS = 
-LIBOBJS = 
-LIBS = 
-LIBTOOL = $(SHELL) $(top_builddir)/libtool
-LIPO = 
-LN_S = ln -s
-LTLIBOBJS = 
-MAINT = #
-MAKEINFO = ${SHELL} /home/rich/src/gcc/missing --run makeinfo
-MKDIR_P = /usr/bin/mkdir -p
-NM = /usr/bin/nm -B
-NMEDIT = 
-OBJDUMP = objdump
-OBJEXT = o
-OTOOL = 
-OTOOL64 = 
-PACKAGE = libbacktrace
-PACKAGE_BUGREPORT = 
-PACKAGE_NAME = package-unused
-PACKAGE_STRING = package-unused version-unused
-PACKAGE_TARNAME = libbacktrace
-PACKAGE_URL = 
-PACKAGE_VERSION = version-unused
-PATH_SEPARATOR = :
-PIC_FLAG = 
-RANLIB = ranlib
-SED = /usr/bin/sed
-SET_MAKE = 
-SHELL = /bin/sh
-STRIP = strip
-VERSION = version-unused
-VIEW_FILE = mmapio.lo
-WARN_FLAGS = -W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual
-abs_builddir = /home/rich/src/gcc/libbacktrace
-abs_srcdir = /home/rich/src/gcc/libbacktrace
-abs_top_builddir = /home/rich/src/gcc/libbacktrace
-abs_top_srcdir = /home/rich/src/gcc/libbacktrace
-ac_ct_CC = gcc
-ac_ct_DUMPBIN = 
-am__leading_dot = .
-am__tar = $${TAR-tar} chof - "$$tardir"
-am__untar = $${TAR-tar} xf -
-bindir = ${exec_prefix}/bin
-build = x86_64-pc-linux-gnu
-build_alias = 
-build_cpu = x86_64
-build_os = linux-gnu
-build_vendor = pc
-builddir = .
-datadir = ${datarootdir}
-datarootdir = ${prefix}/share
-docdir = ${datarootdir}/doc/${PACKAGE_TARNAME}
-dvidir = ${docdir}
-exec_prefix = ${prefix}
-host = x86_64-pc-linux-gnu
-host_alias = 
-host_cpu = x86_64
-host_os = linux-gnu
-host_vendor = pc
-htmldir = ${docdir}
-includedir = ${prefix}/include
-infodir = ${datarootdir}/info
-install_sh = ${SHELL} /home/rich/src/gcc/install-sh
-libdir = ${exec_prefix}/lib
-libexecdir = ${exec_prefix}/libexec
-libtool_VERSION = 1:0:0
-localedir = ${datarootdir}/locale
-localstatedir = ${prefix}/var
-mandir = ${datarootdir}/man
-mkdir_p = /usr/bin/mkdir -p
-multi_basedir = 
-oldincludedir = /usr/include
-pdfdir = ${docdir}
-prefix = /usr/local
-program_transform_name = s,x,x,
-psdir = ${docdir}
-sbindir = ${exec_prefix}/sbin
-sharedstatedir = ${prefix}/com
-srcdir = .
-sysconfdir = ${prefix}/etc
-target = x86_64-pc-linux-gnu
-target_alias = 
-target_cpu = x86_64
-target_os = linux-gnu
-target_vendor = pc
-top_build_prefix = 
-top_builddir = .
-top_srcdir = .
-ACLOCAL_AMFLAGS = -I .. -I ../config
-AM_CPPFLAGS = -I $(top_srcdir)/../include -I $(top_srcdir)/../libgcc \
-	-I ../libgcc
-
-AM_CFLAGS = $(EXTRA_FLAGS) $(WARN_FLAGS) $(PIC_FLAG)
-noinst_LTLIBRARIES = libbacktrace.la
-libbacktrace_la_SOURCES = \
-	backtrace.h \
-	atomic.c \
-	dwarf.c \
-	fileline.c \
-	internal.h \
-	posix.c \
-	print.c \
-	sort.c \
-	state.c
-
-BACKTRACE_FILES = \
-	backtrace.c \
-	simple.c \
-	nounwind.c
-
-FORMAT_FILES = \
-	elf.c \
-	pecoff.c \
-	unknown.c
-
-VIEW_FILES = \
-	read.c \
-	mmapio.c
-
-ALLOC_FILES = \
-	alloc.c \
-	mmap.c
-
-EXTRA_libbacktrace_la_SOURCES = \
-	$(BACKTRACE_FILES) \
-	$(FORMAT_FILES) \
-	$(VIEW_FILES) \
-	$(ALLOC_FILES)
-
-libbacktrace_la_LIBADD = \
-	$(BACKTRACE_FILE) \
-	$(FORMAT_FILE) \
-	$(VIEW_FILE) \
-	$(ALLOC_FILE)
-
-libbacktrace_la_DEPENDENCIES = $(libbacktrace_la_LIBADD)
-TESTS = $(check_PROGRAMS)
-btest_SOURCES = btest.c
-btest_CFLAGS = $(AM_CFLAGS) -g -O
-btest_LDADD = libbacktrace.la
-stest_SOURCES = stest.c
-stest_LDADD = libbacktrace.la
-
-# We can't use automake's automatic dependency tracking, because it
-# breaks when using bootstrap-lean.  Automatic dependency tracking
-# with GCC bootstrap will cause some of the objects to depend on
-# header files in prev-gcc/include, e.g., stddef.h and stdarg.h.  When
-# using bootstrap-lean, prev-gcc is removed after each stage.  When
-# running "make install", those header files will be gone, causing the
-# library to be rebuilt at install time.  That may not succeed.
-
-# These manual dependencies do not include dependencies on unwind.h,
-# even though that is part of GCC, because where to find it depends on
-# whether we are being built as a host library or a target library.
-INCDIR = $(top_srcdir)/../include
-all: config.h
-	$(MAKE) $(AM_MAKEFLAGS) all-am
-
-.SUFFIXES:
-.SUFFIXES: .c .lo .o .obj
-am--refresh: Makefile
-	@:
-$(srcdir)/Makefile.in: # $(srcdir)/Makefile.am  $(am__configure_deps)
-	@for dep in $?; do \
-	  case '$(am__configure_deps)' in \
-	    *$$dep*) \
-	      echo ' cd $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps'; \
-	      $(am__cd) $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps \
-		&& exit 0; \
-	      exit 1;; \
-	  esac; \
-	done; \
-	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign --ignore-deps Makefile'; \
-	$(am__cd) $(top_srcdir) && \
-	  $(AUTOMAKE) --foreign --ignore-deps Makefile
-.PRECIOUS: Makefile
-Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
-	@case '$?' in \
-	  *config.status*) \
-	    echo ' $(SHELL) ./config.status'; \
-	    $(SHELL) ./config.status;; \
-	  *) \
-	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \
-	    cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \
-	esac;
-
-$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
-	$(SHELL) ./config.status --recheck
-
-$(top_srcdir)/configure: # $(am__configure_deps)
-	$(am__cd) $(srcdir) && $(AUTOCONF)
-$(ACLOCAL_M4): # $(am__aclocal_m4_deps)
-	$(am__cd) $(srcdir) && $(ACLOCAL) $(ACLOCAL_AMFLAGS)
-$(am__aclocal_m4_deps):
-
-config.h: stamp-h1
-	@if test ! -f $@; then rm -f stamp-h1; else :; fi
-	@if test ! -f $@; then $(MAKE) $(AM_MAKEFLAGS) stamp-h1; else :; fi
-
-stamp-h1: $(srcdir)/config.h.in $(top_builddir)/config.status
-	@rm -f stamp-h1
-	cd $(top_builddir) && $(SHELL) ./config.status config.h
-$(srcdir)/config.h.in: # $(am__configure_deps) 
-	($(am__cd) $(top_srcdir) && $(AUTOHEADER))
-	rm -f stamp-h1
-	touch $@
-
-distclean-hdr:
-	-rm -f config.h stamp-h1
-backtrace-supported.h: $(top_builddir)/config.status $(srcdir)/backtrace-supported.h.in
-	cd $(top_builddir) && $(SHELL) ./config.status $@
-
-clean-noinstLTLIBRARIES:
-	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
-	@list='$(noinst_LTLIBRARIES)'; for p in $$list; do \
-	  dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
-	  test "$$dir" != "$$p" || dir=.; \
-	  echo "rm -f \"$${dir}/so_locations\""; \
-	  rm -f "$${dir}/so_locations"; \
-	done
-libbacktrace.la: $(libbacktrace_la_OBJECTS) $(libbacktrace_la_DEPENDENCIES) $(EXTRA_libbacktrace_la_DEPENDENCIES) 
-	$(LINK)  $(libbacktrace_la_OBJECTS) $(libbacktrace_la_LIBADD) $(LIBS)
-
-clean-checkPROGRAMS:
-	@list='$(check_PROGRAMS)'; test -n "$$list" || exit 0; \
-	echo " rm -f" $$list; \
-	rm -f $$list || exit $$?; \
-	test -n "$(EXEEXT)" || exit 0; \
-	list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
-	echo " rm -f" $$list; \
-	rm -f $$list
-btest$(EXEEXT): $(btest_OBJECTS) $(btest_DEPENDENCIES) $(EXTRA_btest_DEPENDENCIES) 
-	@rm -f btest$(EXEEXT)
-	$(btest_LINK) $(btest_OBJECTS) $(btest_LDADD) $(LIBS)
-stest$(EXEEXT): $(stest_OBJECTS) $(stest_DEPENDENCIES) $(EXTRA_stest_DEPENDENCIES) 
-	@rm -f stest$(EXEEXT)
-	$(LINK) $(stest_OBJECTS) $(stest_LDADD) $(LIBS)
-
-mostlyclean-compile:
-	-rm -f *.$(OBJEXT)
-
-distclean-compile:
-	-rm -f *.tab.c
-
-.c.o:
-	$(COMPILE) -c $<
-
-.c.obj:
-	$(COMPILE) -c `$(CYGPATH_W) '$<'`
-
-.c.lo:
-	$(LTCOMPILE) -c -o $@ $<
-
-btest-btest.o: btest.c
-	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(btest_CFLAGS) $(CFLAGS) -c -o btest-btest.o `test -f 'btest.c' || echo '$(srcdir)/'`btest.c
-
-btest-btest.obj: btest.c
-	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(btest_CFLAGS) $(CFLAGS) -c -o btest-btest.obj `if test -f 'btest.c'; then $(CYGPATH_W) 'btest.c'; else $(CYGPATH_W) '$(srcdir)/btest.c'; fi`
-
-mostlyclean-libtool:
-	-rm -f *.lo
-
-clean-libtool:
-	-rm -rf .libs _libs
-
-distclean-libtool:
-	-rm -f libtool config.lt
-
-# GNU Make needs to see an explicit $(MAKE) variable in the command it
-# runs to enable its job server during parallel builds.  Hence the
-# comments below.
-all-multi:
-	$(MULTIDO) $(AM_MAKEFLAGS) DO=all multi-do # $(MAKE)
-install-multi:
-	$(MULTIDO) $(AM_MAKEFLAGS) DO=install multi-do # $(MAKE)
-
-mostlyclean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=mostlyclean multi-clean # $(MAKE)
-clean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=clean multi-clean # $(MAKE)
-distclean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=distclean multi-clean # $(MAKE)
-maintainer-clean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=maintainer-clean multi-clean # $(MAKE)
-
-ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
-	list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
-	unique=`for i in $$list; do \
-	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
-	  done | \
-	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
-	      END { if (nonempty) { for (i in files) print i; }; }'`; \
-	mkid -fID $$unique
-tags: TAGS
-
-TAGS:  $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
-		$(TAGS_FILES) $(LISP)
-	set x; \
-	here=`pwd`; \
-	list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
-	unique=`for i in $$list; do \
-	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
-	  done | \
-	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
-	      END { if (nonempty) { for (i in files) print i; }; }'`; \
-	shift; \
-	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
-	  test -n "$$unique" || unique=$$empty_fix; \
-	  if test $$# -gt 0; then \
-	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
-	      "$$@" $$unique; \
-	  else \
-	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
-	      $$unique; \
-	  fi; \
-	fi
-ctags: CTAGS
-CTAGS:  $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
-		$(TAGS_FILES) $(LISP)
-	list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
-	unique=`for i in $$list; do \
-	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
-	  done | \
-	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
-	      END { if (nonempty) { for (i in files) print i; }; }'`; \
-	test -z "$(CTAGS_ARGS)$$unique" \
-	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
-	     $$unique
-
-GTAGS:
-	here=`$(am__cd) $(top_builddir) && pwd` \
-	  && $(am__cd) $(top_srcdir) \
-	  && gtags -i $(GTAGS_ARGS) "$$here"
-
-distclean-tags:
-	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
-
-check-TESTS: $(TESTS)
-	@failed=0; all=0; xfail=0; xpass=0; skip=0; \
-	srcdir=$(srcdir); export srcdir; \
-	list=' $(TESTS) '; \
-	$(am__tty_colors); \
-	if test -n "$$list"; then \
-	  for tst in $$list; do \
-	    if test -f ./$$tst; then dir=./; \
-	    elif test -f $$tst; then dir=; \
-	    else dir="$(srcdir)/"; fi; \
-	    if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
-	      all=`expr $$all + 1`; \
-	      case " $(XFAIL_TESTS) " in \
-	      *[\ \	]$$tst[\ \	]*) \
-		xpass=`expr $$xpass + 1`; \
-		failed=`expr $$failed + 1`; \
-		col=$$red; res=XPASS; \
-	      ;; \
-	      *) \
-		col=$$grn; res=PASS; \
-	      ;; \
-	      esac; \
-	    elif test $$? -ne 77; then \
-	      all=`expr $$all + 1`; \
-	      case " $(XFAIL_TESTS) " in \
-	      *[\ \	]$$tst[\ \	]*) \
-		xfail=`expr $$xfail + 1`; \
-		col=$$lgn; res=XFAIL; \
-	      ;; \
-	      *) \
-		failed=`expr $$failed + 1`; \
-		col=$$red; res=FAIL; \
-	      ;; \
-	      esac; \
-	    else \
-	      skip=`expr $$skip + 1`; \
-	      col=$$blu; res=SKIP; \
-	    fi; \
-	    echo "$${col}$$res$${std}: $$tst"; \
-	  done; \
-	  if test "$$all" -eq 1; then \
-	    tests="test"; \
-	    All=""; \
-	  else \
-	    tests="tests"; \
-	    All="All "; \
-	  fi; \
-	  if test "$$failed" -eq 0; then \
-	    if test "$$xfail" -eq 0; then \
-	      banner="$$All$$all $$tests passed"; \
-	    else \
-	      if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \
-	      banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \
-	    fi; \
-	  else \
-	    if test "$$xpass" -eq 0; then \
-	      banner="$$failed of $$all $$tests failed"; \
-	    else \
-	      if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \
-	      banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \
-	    fi; \
-	  fi; \
-	  dashes="$$banner"; \
-	  skipped=""; \
-	  if test "$$skip" -ne 0; then \
-	    if test "$$skip" -eq 1; then \
-	      skipped="($$skip test was not run)"; \
-	    else \
-	      skipped="($$skip tests were not run)"; \
-	    fi; \
-	    test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \
-	      dashes="$$skipped"; \
-	  fi; \
-	  report=""; \
-	  if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \
-	    report="Please report to $(PACKAGE_BUGREPORT)"; \
-	    test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \
-	      dashes="$$report"; \
-	  fi; \
-	  dashes=`echo "$$dashes" | sed s/./=/g`; \
-	  if test "$$failed" -eq 0; then \
-	    col="$$grn"; \
-	  else \
-	    col="$$red"; \
-	  fi; \
-	  echo "$${col}$$dashes$${std}"; \
-	  echo "$${col}$$banner$${std}"; \
-	  test -z "$$skipped" || echo "$${col}$$skipped$${std}"; \
-	  test -z "$$report" || echo "$${col}$$report$${std}"; \
-	  echo "$${col}$$dashes$${std}"; \
-	  test "$$failed" -eq 0; \
-	else :; fi
-check-am: all-am
-	$(MAKE) $(AM_MAKEFLAGS) $(check_PROGRAMS)
-	$(MAKE) $(AM_MAKEFLAGS) check-TESTS
-check: check-am
-all-am: Makefile $(LTLIBRARIES) all-multi config.h
-installdirs:
-install: install-am
-install-exec: install-exec-am
-install-data: install-data-am
-uninstall: uninstall-am
-
-install-am: all-am
-	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
-
-installcheck: installcheck-am
-install-strip:
-	if test -z '$(STRIP)'; then \
-	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
-	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
-	      install; \
-	else \
-	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
-	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
-	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
-	fi
-mostlyclean-generic:
-
-clean-generic:
-
-distclean-generic:
-	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
-	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
-
-maintainer-clean-generic:
-	@echo "This command is intended for maintainers to use"
-	@echo "it deletes files that may require special tools to rebuild."
-clean: clean-am clean-multi
-
-clean-am: clean-checkPROGRAMS clean-generic clean-libtool \
-	clean-noinstLTLIBRARIES mostlyclean-am
-
-distclean: distclean-am distclean-multi
-	-rm -f $(am__CONFIG_DISTCLEAN_FILES)
-	-rm -f Makefile
-distclean-am: clean-am distclean-compile distclean-generic \
-	distclean-hdr distclean-libtool distclean-tags
-
-dvi: dvi-am
-
-dvi-am:
-
-html: html-am
-
-html-am:
-
-info: info-am
-
-info-am:
-
-install-data-am:
-
-install-dvi: install-dvi-am
-
-install-dvi-am:
-
-install-exec-am: install-multi
-
-install-html: install-html-am
-
-install-html-am:
-
-install-info: install-info-am
-
-install-info-am:
-
-install-man:
-
-install-pdf: install-pdf-am
-
-install-pdf-am:
-
-install-ps: install-ps-am
-
-install-ps-am:
-
-installcheck-am:
-
-maintainer-clean: maintainer-clean-am maintainer-clean-multi
-	-rm -f $(am__CONFIG_DISTCLEAN_FILES)
-	-rm -rf $(top_srcdir)/autom4te.cache
-	-rm -f Makefile
-maintainer-clean-am: distclean-am maintainer-clean-generic
-
-mostlyclean: mostlyclean-am mostlyclean-multi
-
-mostlyclean-am: mostlyclean-compile mostlyclean-generic \
-	mostlyclean-libtool
-
-pdf: pdf-am
-
-pdf-am:
-
-ps: ps-am
-
-ps-am:
-
-uninstall-am:
-
-.MAKE: all all-multi check-am clean-multi distclean-multi install-am \
-	install-multi install-strip maintainer-clean-multi \
-	mostlyclean-multi
-
-.PHONY: CTAGS GTAGS all all-am all-multi am--refresh check check-TESTS \
-	check-am clean clean-checkPROGRAMS clean-generic clean-libtool \
-	clean-multi clean-noinstLTLIBRARIES ctags distclean \
-	distclean-compile distclean-generic distclean-hdr \
-	distclean-libtool distclean-multi distclean-tags dvi dvi-am \
-	html html-am info info-am install install-am install-data \
-	install-data-am install-dvi install-dvi-am install-exec \
-	install-exec-am install-html install-html-am install-info \
-	install-info-am install-man install-multi install-pdf \
-	install-pdf-am install-ps install-ps-am install-strip \
-	installcheck installcheck-am installdirs maintainer-clean \
-	maintainer-clean-generic maintainer-clean-multi mostlyclean \
-	mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
-	mostlyclean-multi pdf pdf-am ps ps-am tags uninstall \
-	uninstall-am
-
-alloc.lo: config.h backtrace.h internal.h
-backtrace.lo: config.h backtrace.h internal.h
-btest.lo: (INCDIR)/filenames.h backtrace.h backtrace-supported.h
-dwarf.lo: config.h $(INCDIR)/dwarf2.h $(INCDIR)/dwarf2.def \
-	$(INCDIR)/filenames.h backtrace.h internal.h
-elf.lo: config.h backtrace.h internal.h
-fileline.lo: config.h backtrace.h internal.h
-mmap.lo: config.h backtrace.h internal.h
-mmapio.lo: config.h backtrace.h internal.h
-nounwind.lo: config.h internal.h
-pecoff.lo: config.h backtrace.h internal.h
-posix.lo: config.h backtrace.h internal.h
-print.lo: config.h backtrace.h internal.h
-read.lo: config.h backtrace.h internal.h
-simple.lo: config.h backtrace.h internal.h
-sort.lo: config.h backtrace.h internal.h
-stest.lo: config.h backtrace.h internal.h
-state.lo: config.h backtrace.h backtrace-supported.h internal.h
-unknown.lo: config.h backtrace.h internal.h
-
-# Tell versions [3.59,3.63) of GNU make to not export all variables.
-# Otherwise a system limit (for SysV at least) may be exceeded.
-.NOEXPORT:
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/Makefile.am b/rpython/rlib/rvmprof/src/libbacktrace/Makefile.am
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/libbacktrace/Makefile.am
+++ /dev/null
@@ -1,136 +0,0 @@
-# Makefile.am -- Backtrace Makefile.
-# Copyright (C) 2012-2017 Free Software Foundation, Inc.
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-
-#     (1) Redistributions of source code must retain the above copyright
-#     notice, this list of conditions and the following disclaimer.
-
-#     (2) Redistributions in binary form must reproduce the above copyright
-#     notice, this list of conditions and the following disclaimer in
-#     the documentation and/or other materials provided with the
-#     distribution.
-
-#     (3) The name of the author may not be used to
-#     endorse or promote products derived from this software without
-#     specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-ACLOCAL_AMFLAGS = -I .. -I ../config
-
-AM_CPPFLAGS = -I $(top_srcdir)/../include -I $(top_srcdir)/../libgcc \
-	-I ../libgcc
-
-AM_CFLAGS = $(EXTRA_FLAGS) $(WARN_FLAGS) $(PIC_FLAG)
-
-noinst_LTLIBRARIES = libbacktrace.la
-
-libbacktrace_la_SOURCES = \
-	backtrace.h \
-	atomic.c \
-	dwarf.c \
-	fileline.c \
-	internal.h \
-	posix.c \
-	print.c \
-	sort.c \
-	state.c
-
-BACKTRACE_FILES = \
-	backtrace.c \
-	simple.c \
-	nounwind.c
-
-FORMAT_FILES = \
-	elf.c \
-	pecoff.c \
-	unknown.c
-
-VIEW_FILES = \
-	read.c \
-	mmapio.c
-
-ALLOC_FILES = \
-	alloc.c \
-	mmap.c
-
-EXTRA_libbacktrace_la_SOURCES = \
-	$(BACKTRACE_FILES) \
-	$(FORMAT_FILES) \
-	$(VIEW_FILES) \
-	$(ALLOC_FILES)
-
-libbacktrace_la_LIBADD = \
-	$(BACKTRACE_FILE) \
-	$(FORMAT_FILE) \
-	$(VIEW_FILE) \
-	$(ALLOC_FILE)
-
-libbacktrace_la_DEPENDENCIES = $(libbacktrace_la_LIBADD)
-
-# Testsuite.
-
-check_PROGRAMS =
-
-TESTS = $(check_PROGRAMS)
-
-if NATIVE
-
-btest_SOURCES = btest.c
-btest_CFLAGS = $(AM_CFLAGS) -g -O
-btest_LDADD = libbacktrace.la
-
-check_PROGRAMS += btest
-
-stest_SOURCES = stest.c
-stest_LDADD = libbacktrace.la
-
-check_PROGRAMS += stest
-
-endif NATIVE
-
-# We can't use automake's automatic dependency tracking, because it
-# breaks when using bootstrap-lean.  Automatic dependency tracking
-# with GCC bootstrap will cause some of the objects to depend on
-# header files in prev-gcc/include, e.g., stddef.h and stdarg.h.  When
-# using bootstrap-lean, prev-gcc is removed after each stage.  When
-# running "make install", those header files will be gone, causing the
-# library to be rebuilt at install time.  That may not succeed.
-
-# These manual dependencies do not include dependencies on unwind.h,
-# even though that is part of GCC, because where to find it depends on
-# whether we are being built as a host library or a target library.
-
-INCDIR = $(top_srcdir)/../include
-alloc.lo: config.h backtrace.h internal.h
-backtrace.lo: config.h backtrace.h internal.h
-btest.lo: (INCDIR)/filenames.h backtrace.h backtrace-supported.h
-dwarf.lo: config.h $(INCDIR)/dwarf2.h $(INCDIR)/dwarf2.def \
-	$(INCDIR)/filenames.h backtrace.h internal.h
-elf.lo: config.h backtrace.h internal.h
-fileline.lo: config.h backtrace.h internal.h
-mmap.lo: config.h backtrace.h internal.h
-mmapio.lo: config.h backtrace.h internal.h
-nounwind.lo: config.h internal.h
-pecoff.lo: config.h backtrace.h internal.h
-posix.lo: config.h backtrace.h internal.h
-print.lo: config.h backtrace.h internal.h
-read.lo: config.h backtrace.h internal.h
-simple.lo: config.h backtrace.h internal.h
-sort.lo: config.h backtrace.h internal.h
-stest.lo: config.h backtrace.h internal.h
-state.lo: config.h backtrace.h backtrace-supported.h internal.h
-unknown.lo: config.h backtrace.h internal.h
diff --git a/rpython/rlib/rvmprof/src/libbacktrace/Makefile.in b/rpython/rlib/rvmprof/src/libbacktrace/Makefile.in
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/libbacktrace/Makefile.in
+++ /dev/null
@@ -1,770 +0,0 @@
-# Makefile.in generated by automake 1.11.6 from Makefile.am.
-# @configure_input@
-
-# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
-# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-# Foundation, Inc.
-# This Makefile.in is free software; the Free Software Foundation
-# gives unlimited permission to copy and/or distribute it,
-# with or without modifications, as long as this notice is preserved.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
-# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE.
-
- at SET_MAKE@
-
-# Makefile.am -- Backtrace Makefile.
-# Copyright (C) 2012-2016 Free Software Foundation, Inc.
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-
-#     (1) Redistributions of source code must retain the above copyright
-#     notice, this list of conditions and the following disclaimer.
-
-#     (2) Redistributions in binary form must reproduce the above copyright
-#     notice, this list of conditions and the following disclaimer in
-#     the documentation and/or other materials provided with the
-#     distribution.
-
-#     (3) The name of the author may not be used to
-#     endorse or promote products derived from this software without
-#     specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-VPATH = @srcdir@
-am__make_dryrun = \
-  { \
-    am__dry=no; \
-    case $$MAKEFLAGS in \
-      *\\[\ \	]*) \
-        echo 'am--echo: ; @echo "AM"  OK' | $(MAKE) -f - 2>/dev/null \
-          | grep '^AM OK$$' >/dev/null || am__dry=yes;; \
-      *) \
-        for am__flg in $$MAKEFLAGS; do \
-          case $$am__flg in \
-            *=*|--*) ;; \
-            *n*) am__dry=yes; break;; \
-          esac; \
-        done;; \
-    esac; \
-    test $$am__dry = yes; \
-  }
-pkgdatadir = $(datadir)/@PACKAGE@
-pkgincludedir = $(includedir)/@PACKAGE@
-pkglibdir = $(libdir)/@PACKAGE@
-pkglibexecdir = $(libexecdir)/@PACKAGE@
-am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
-install_sh_DATA = $(install_sh) -c -m 644
-install_sh_PROGRAM = $(install_sh) -c
-install_sh_SCRIPT = $(install_sh) -c
-INSTALL_HEADER = $(INSTALL_DATA)
-transform = $(program_transform_name)
-NORMAL_INSTALL = :
-PRE_INSTALL = :
-POST_INSTALL = :
-NORMAL_UNINSTALL = :
-PRE_UNINSTALL = :
-POST_UNINSTALL = :
-build_triplet = @build@
-host_triplet = @host@
-target_triplet = @target@
-check_PROGRAMS = $(am__EXEEXT_1)
- at NATIVE_TRUE@am__append_1 = btest stest
-subdir = .
-DIST_COMMON = README ChangeLog $(srcdir)/Makefile.in \
-	$(srcdir)/Makefile.am $(top_srcdir)/configure \
-	$(am__configure_deps) $(srcdir)/config.h.in \
-	$(srcdir)/../mkinstalldirs $(srcdir)/backtrace-supported.h.in
-ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/../config/lead-dot.m4 \
-	$(top_srcdir)/../config/multi.m4 \
-	$(top_srcdir)/../config/override.m4 \
-	$(top_srcdir)/../config/stdint.m4 \
-	$(top_srcdir)/../config/unwind_ipinfo.m4 \
-	$(top_srcdir)/../config/warnings.m4 \
-	$(top_srcdir)/../libtool.m4 $(top_srcdir)/../ltoptions.m4 \
-	$(top_srcdir)/../ltsugar.m4 $(top_srcdir)/../ltversion.m4 \
-	$(top_srcdir)/../lt~obsolete.m4 $(top_srcdir)/configure.ac
-am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
-	$(ACLOCAL_M4)
-am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
- configure.lineno config.status.lineno
-mkinstalldirs = $(SHELL) $(top_srcdir)/../mkinstalldirs
-CONFIG_HEADER = config.h
-CONFIG_CLEAN_FILES = backtrace-supported.h
-CONFIG_CLEAN_VPATH_FILES =
-LTLIBRARIES = $(noinst_LTLIBRARIES)
-am__DEPENDENCIES_1 =
-am_libbacktrace_la_OBJECTS = atomic.lo dwarf.lo fileline.lo posix.lo \
-	print.lo sort.lo state.lo
-libbacktrace_la_OBJECTS = $(am_libbacktrace_la_OBJECTS)
- at NATIVE_TRUE@am__EXEEXT_1 = btest$(EXEEXT) stest$(EXEEXT)
- at NATIVE_TRUE@am_btest_OBJECTS = btest-btest.$(OBJEXT)
-btest_OBJECTS = $(am_btest_OBJECTS)
- at NATIVE_TRUE@btest_DEPENDENCIES = libbacktrace.la
-btest_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=link $(CCLD) $(btest_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
-	$(LDFLAGS) -o $@
- at NATIVE_TRUE@am_stest_OBJECTS = stest.$(OBJEXT)
-stest_OBJECTS = $(am_stest_OBJECTS)
- at NATIVE_TRUE@stest_DEPENDENCIES = libbacktrace.la
-DEFAULT_INCLUDES = -I. at am__isrc@
-depcomp =
-am__depfiles_maybe =
-COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
-	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
-LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
-	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
-CCLD = $(CC)
-LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
-	$(LDFLAGS) -o $@
-SOURCES = $(libbacktrace_la_SOURCES) $(EXTRA_libbacktrace_la_SOURCES) \
-	$(btest_SOURCES) $(stest_SOURCES)
-MULTISRCTOP = 
-MULTIBUILDTOP = 
-MULTIDIRS = 
-MULTISUBDIR = 
-MULTIDO = true
-MULTICLEAN = true
-am__can_run_installinfo = \
-  case $$AM_UPDATE_INFO_DIR in \
-    n|no|NO) false;; \
-    *) (install-info --version) >/dev/null 2>&1;; \
-  esac
-ETAGS = etags
-CTAGS = ctags
-am__tty_colors = \
-red=; grn=; lgn=; blu=; std=
-ACLOCAL = @ACLOCAL@
-ALLOC_FILE = @ALLOC_FILE@
-AMTAR = @AMTAR@
-AR = @AR@
-AUTOCONF = @AUTOCONF@
-AUTOHEADER = @AUTOHEADER@
-AUTOMAKE = @AUTOMAKE@
-AWK = @AWK@
-BACKTRACE_FILE = @BACKTRACE_FILE@
-BACKTRACE_SUPPORTED = @BACKTRACE_SUPPORTED@
-BACKTRACE_SUPPORTS_DATA = @BACKTRACE_SUPPORTS_DATA@
-BACKTRACE_SUPPORTS_THREADS = @BACKTRACE_SUPPORTS_THREADS@
-BACKTRACE_USES_MALLOC = @BACKTRACE_USES_MALLOC@
-CC = @CC@
-CFLAGS = @CFLAGS@
-CPP = @CPP@
-CPPFLAGS = @CPPFLAGS@
-CYGPATH_W = @CYGPATH_W@
-DEFS = @DEFS@
-DSYMUTIL = @DSYMUTIL@
-DUMPBIN = @DUMPBIN@
-ECHO_C = @ECHO_C@
-ECHO_N = @ECHO_N@
-ECHO_T = @ECHO_T@
-EGREP = @EGREP@
-EXEEXT = @EXEEXT@
-EXTRA_FLAGS = @EXTRA_FLAGS@
-FGREP = @FGREP@
-FORMAT_FILE = @FORMAT_FILE@
-GREP = @GREP@
-INSTALL = @INSTALL@
-INSTALL_DATA = @INSTALL_DATA@
-INSTALL_PROGRAM = @INSTALL_PROGRAM@
-INSTALL_SCRIPT = @INSTALL_SCRIPT@
-INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
-LD = @LD@
-LDFLAGS = @LDFLAGS@
-LIBOBJS = @LIBOBJS@
-LIBS = @LIBS@
-LIBTOOL = @LIBTOOL@
-LIPO = @LIPO@
-LN_S = @LN_S@
-LTLIBOBJS = @LTLIBOBJS@
-MAINT = @MAINT@
-MAKEINFO = @MAKEINFO@
-MKDIR_P = @MKDIR_P@
-NM = @NM@
-NMEDIT = @NMEDIT@
-OBJDUMP = @OBJDUMP@
-OBJEXT = @OBJEXT@
-OTOOL = @OTOOL@
-OTOOL64 = @OTOOL64@
-PACKAGE = @PACKAGE@
-PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
-PACKAGE_NAME = @PACKAGE_NAME@
-PACKAGE_STRING = @PACKAGE_STRING@
-PACKAGE_TARNAME = @PACKAGE_TARNAME@
-PACKAGE_URL = @PACKAGE_URL@
-PACKAGE_VERSION = @PACKAGE_VERSION@
-PATH_SEPARATOR = @PATH_SEPARATOR@
-PIC_FLAG = @PIC_FLAG@
-RANLIB = @RANLIB@
-SED = @SED@
-SET_MAKE = @SET_MAKE@
-SHELL = @SHELL@
-STRIP = @STRIP@
-VERSION = @VERSION@
-VIEW_FILE = @VIEW_FILE@
-WARN_FLAGS = @WARN_FLAGS@
-abs_builddir = @abs_builddir@
-abs_srcdir = @abs_srcdir@
-abs_top_builddir = @abs_top_builddir@
-abs_top_srcdir = @abs_top_srcdir@
-ac_ct_CC = @ac_ct_CC@
-ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
-am__leading_dot = @am__leading_dot@
-am__tar = @am__tar@
-am__untar = @am__untar@
-bindir = @bindir@
-build = @build@
-build_alias = @build_alias@
-build_cpu = @build_cpu@
-build_os = @build_os@
-build_vendor = @build_vendor@
-builddir = @builddir@
-datadir = @datadir@
-datarootdir = @datarootdir@
-docdir = @docdir@
-dvidir = @dvidir@
-exec_prefix = @exec_prefix@
-host = @host@
-host_alias = @host_alias@
-host_cpu = @host_cpu@
-host_os = @host_os@
-host_vendor = @host_vendor@
-htmldir = @htmldir@
-includedir = @includedir@
-infodir = @infodir@
-install_sh = @install_sh@
-libdir = @libdir@
-libexecdir = @libexecdir@
-libtool_VERSION = @libtool_VERSION@
-localedir = @localedir@
-localstatedir = @localstatedir@
-mandir = @mandir@
-mkdir_p = @mkdir_p@
-multi_basedir = @multi_basedir@
-oldincludedir = @oldincludedir@
-pdfdir = @pdfdir@
-prefix = @prefix@
-program_transform_name = @program_transform_name@
-psdir = @psdir@
-sbindir = @sbindir@
-sharedstatedir = @sharedstatedir@
-srcdir = @srcdir@
-sysconfdir = @sysconfdir@
-target = @target@
-target_alias = @target_alias@
-target_cpu = @target_cpu@
-target_os = @target_os@
-target_vendor = @target_vendor@
-top_build_prefix = @top_build_prefix@
-top_builddir = @top_builddir@
-top_srcdir = @top_srcdir@
-ACLOCAL_AMFLAGS = -I .. -I ../config
-AM_CPPFLAGS = -I $(top_srcdir)/../include -I $(top_srcdir)/../libgcc \
-	-I ../libgcc
-
-AM_CFLAGS = $(EXTRA_FLAGS) $(WARN_FLAGS) $(PIC_FLAG)
-noinst_LTLIBRARIES = libbacktrace.la
-libbacktrace_la_SOURCES = \
-	backtrace.h \
-	atomic.c \
-	dwarf.c \
-	fileline.c \
-	internal.h \
-	posix.c \
-	print.c \
-	sort.c \
-	state.c
-
-BACKTRACE_FILES = \
-	backtrace.c \
-	simple.c \
-	nounwind.c
-
-FORMAT_FILES = \
-	elf.c \
-	pecoff.c \
-	unknown.c
-
-VIEW_FILES = \
-	read.c \
-	mmapio.c
-
-ALLOC_FILES = \
-	alloc.c \
-	mmap.c
-
-EXTRA_libbacktrace_la_SOURCES = \
-	$(BACKTRACE_FILES) \
-	$(FORMAT_FILES) \
-	$(VIEW_FILES) \
-	$(ALLOC_FILES)
-
-libbacktrace_la_LIBADD = \
-	$(BACKTRACE_FILE) \
-	$(FORMAT_FILE) \
-	$(VIEW_FILE) \
-	$(ALLOC_FILE)
-
-libbacktrace_la_DEPENDENCIES = $(libbacktrace_la_LIBADD)
-TESTS = $(check_PROGRAMS)
- at NATIVE_TRUE@btest_SOURCES = btest.c
- at NATIVE_TRUE@btest_CFLAGS = $(AM_CFLAGS) -g -O
- at NATIVE_TRUE@btest_LDADD = libbacktrace.la
- at NATIVE_TRUE@stest_SOURCES = stest.c
- at NATIVE_TRUE@stest_LDADD = libbacktrace.la
-
-# We can't use automake's automatic dependency tracking, because it
-# breaks when using bootstrap-lean.  Automatic dependency tracking
-# with GCC bootstrap will cause some of the objects to depend on
-# header files in prev-gcc/include, e.g., stddef.h and stdarg.h.  When
-# using bootstrap-lean, prev-gcc is removed after each stage.  When
-# running "make install", those header files will be gone, causing the
-# library to be rebuilt at install time.  That may not succeed.
-
-# These manual dependencies do not include dependencies on unwind.h,
-# even though that is part of GCC, because where to find it depends on
-# whether we are being built as a host library or a target library.
-INCDIR = $(top_srcdir)/../include
-all: config.h
-	$(MAKE) $(AM_MAKEFLAGS) all-am
-
-.SUFFIXES:
-.SUFFIXES: .c .lo .o .obj
-am--refresh: Makefile
-	@:
-$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
-	@for dep in $?; do \
-	  case '$(am__configure_deps)' in \
-	    *$$dep*) \
-	      echo ' cd $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps'; \
-	      $(am__cd) $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps \
-		&& exit 0; \
-	      exit 1;; \
-	  esac; \
-	done; \
-	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign --ignore-deps Makefile'; \
-	$(am__cd) $(top_srcdir) && \
-	  $(AUTOMAKE) --foreign --ignore-deps Makefile
-.PRECIOUS: Makefile
-Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
-	@case '$?' in \
-	  *config.status*) \
-	    echo ' $(SHELL) ./config.status'; \
-	    $(SHELL) ./config.status;; \
-	  *) \

From pypy.commits at gmail.com  Mon Feb  6 12:43:42 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 06 Feb 2017 09:43:42 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: removing duplicate files,
 changes for the first 4 compilation units
Message-ID: <5898b5ce.08061c0a.759cf.1e07@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89976:6d3fd42758cc
Date: 2017-02-06 18:42 +0100
http://bitbucket.org/pypy/pypy/changeset/6d3fd42758cc/

Log:	removing duplicate files, changes for the first 4 compilation units

diff too long, truncating to 2000 out of 56508 lines

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -33,8 +33,10 @@
     ]
     _libs = ['dl']
     compile_extra += ['-DVMPROF_UNIX']
+    compile_extra += ['-DVMPROF_LINUX']
 elif sys.platform == 'darwin':
     compile_extra += ['-DVMPROF_UNIX']
+    compile_extra += ['-DVMPROF_MAC']
     separate_module_files = []
     _libs = []
 else:
@@ -71,7 +73,7 @@
                                   [rffi.INT, rffi.DOUBLE, rffi.INT, rffi.INT,
                                    rffi.CCHARP, rffi.INT],
                                   rffi.CCHARP, compilation_info=eci)
-    vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
+    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT], rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
     vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -2,14 +2,8 @@
 
 #ifdef RPYTHON_LL2CTYPES
    /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
-#ifndef RPY_EXTERN
-#define RPY_EXTERN RPY_EXPORTED
-#endif
-#ifdef _WIN32
-#define RPY_EXPORTED __declspec(dllexport)
-#else
-#define RPY_EXPORTED  extern __attribute__((visibility("default")))
-#endif
+
+static volatile int is_enabled = 0;
 
 #else
 #  include "common_header.h"
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -12,11 +12,20 @@
 #include <stdint.h>
 #endif
 
+#ifndef RPY_EXTERN
+#define RPY_EXTERN RPY_EXPORTED
+#endif
+#ifdef _WIN32
+#define RPY_EXPORTED __declspec(dllexport)
+#else
+#define RPY_EXPORTED  extern __attribute__((visibility("default")))
+#endif
+
 
 RPY_EXTERN char *vmprof_init(int fd, double interval, int memory,
                      int lines, const char *interp_name, int native);
 RPY_EXTERN void vmprof_ignore_signals(int);
-RPY_EXTERN int vmprof_enable(void);
+RPY_EXTERN int vmprof_enable(int memory);
 RPY_EXTERN int vmprof_disable(void);
 RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
 RPY_EXTERN void* vmprof_stack_new(void);
diff --git a/rpython/rlib/rvmprof/src/shared/compat.c b/rpython/rlib/rvmprof/src/shared/compat.c
--- a/rpython/rlib/rvmprof/src/shared/compat.c
+++ b/rpython/rlib/rvmprof/src/shared/compat.c
@@ -1,10 +1,13 @@
 #include "compat.h"
 
-#include "_vmprof.h"
-
+#include <string.h>
+#include <assert.h>
 #if VMPROF_WINDOWS
 #define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
+#else
+#include <time.h>
+#include <sys/time.h>
 #endif
 
 static int _vmp_profile_fileno = -1;
diff --git a/rpython/rlib/rvmprof/src/shared/machine.c b/rpython/rlib/rvmprof/src/shared/machine.c
--- a/rpython/rlib/rvmprof/src/shared/machine.c
+++ b/rpython/rlib/rvmprof/src/shared/machine.c
@@ -1,6 +1,6 @@
 #include "machine.h"
 
-#include "_vmprof.h"
+#include "vmprof.h"
 #include <stdio.h>
 
 int vmp_machine_bits(void)
diff --git a/rpython/rlib/rvmprof/src/shared/src/_vmprof.c b/rpython/rlib/rvmprof/src/shared/src/_vmprof.c
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/src/_vmprof.c
+++ /dev/null
@@ -1,413 +0,0 @@
-/*[clinic input]
-module _vmprof
-[clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b443489e38f2be7d]*/
-
-#define _GNU_SOURCE 1
-
-#include <Python.h>
-#include <frameobject.h>
-#include <signal.h>
-
-#include "_vmprof.h"
-
-static volatile int is_enabled = 0;
-
-#if VMPROF_UNIX
-#include "trampoline.h"
-#include "machine.h"
-#include "symboltable.h"
-#include "vmprof_main.h"
-#else
-#include "vmprof_main_win32.h"
-#endif
-#include "stack.h"
-
-static destructor Original_code_dealloc = 0;
-PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
-
-#ifdef VMPROF_UNIX
-#ifdef __clang__
-__attribute__((disable_tail_calls))
-#elif defined(__GNUC__)
-__attribute__((optimize("O1")))
-#endif
-PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag)
-{
-#ifdef X86_64
-    register PY_STACK_FRAME_T * callee_saved asm("rbx");
-#elif defined(X86_32)
-    register PY_STACK_FRAME_T * callee_saved asm("edi");
-#else
-#    error "platform not supported"
-#endif
-
-    asm volatile(
-#ifdef X86_64
-        "movq %1, %0\t\n"
-#elif defined(X86_32)
-        "mov %1, %0\t\n"
-#else
-#    error "platform not supported"
-#endif
-        : "=r" (callee_saved)
-        : "r" (f) );
-    return _default_eval_loop(f, throwflag);
-}
-#endif
-
-static int emit_code_object(PyCodeObject *co)
-{
-    char buf[MAX_FUNC_NAME + 1];
-    char *co_name, *co_filename;
-    int co_firstlineno;
-    int sz;
-#if PY_MAJOR_VERSION >= 3
-    co_name = PyUnicode_AsUTF8(co->co_name);
-    if (co_name == NULL)
-        return -1;
-    co_filename = PyUnicode_AsUTF8(co->co_filename);
-    if (co_filename == NULL)
-        return -1;
-#else
-    co_name = PyString_AS_STRING(co->co_name);
-    co_filename = PyString_AS_STRING(co->co_filename);
-#endif
-    co_firstlineno = co->co_firstlineno;
-
-    sz = snprintf(buf, MAX_FUNC_NAME / 2, "py:%s", co_name);
-    if (sz < 0) sz = 0;
-    if (sz > MAX_FUNC_NAME / 2) sz = MAX_FUNC_NAME / 2;
-    snprintf(buf + sz, MAX_FUNC_NAME / 2, ":%d:%s", co_firstlineno,
-             co_filename);
-    return vmprof_register_virtual_function(buf, CODE_ADDR_TO_UID(co), 500000);
-}
-
-static int _look_for_code_object(PyObject *o, void *all_codes)
-{
-    if (PyCode_Check(o) && !PySet_Contains((PyObject *)all_codes, o)) {
-        Py_ssize_t i;
-        PyCodeObject *co = (PyCodeObject *)o;
-        if (emit_code_object(co) < 0)
-            return -1;
-        if (PySet_Add((PyObject *)all_codes, o) < 0)
-            return -1;
-
-        /* as a special case, recursively look for and add code
-           objects found in the co_consts.  The problem is that code
-           objects are not created as GC-aware in CPython, so we need
-           to hack like this to hope to find most of them. 
-        */
-        i = PyTuple_Size(co->co_consts);
-        while (i > 0) {
-            --i;
-            if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
-                                      all_codes) < 0)
-                return -1;
-        }
-    }
-    return 0;
-}
-
-static void emit_all_code_objects(void)
-{
-    PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
-    Py_ssize_t i, size;
-
-    gc_module = PyImport_ImportModuleNoBlock("gc");
-    if (gc_module == NULL)
-        goto error;
-
-    lst = PyObject_CallMethod(gc_module, "get_objects", "");
-    if (lst == NULL || !PyList_Check(lst))
-        goto error;
-
-    all_codes = PySet_New(NULL);
-    if (all_codes == NULL)
-        goto error;
-
-    size = PyList_GET_SIZE(lst);
-    for (i = 0; i < size; i++) {
-        PyObject *o = PyList_GET_ITEM(lst, i);
-        if (o->ob_type->tp_traverse &&
-            o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes)
-                < 0)
-            goto error;
-    }
-
- error:
-    Py_XDECREF(all_codes);
-    Py_XDECREF(lst);
-    Py_XDECREF(gc_module);
-}
-
-static void cpyprof_code_dealloc(PyObject *co)
-{
-    if (is_enabled) {
-        emit_code_object((PyCodeObject *)co);
-        /* xxx error return values are ignored */
-    }
-    Original_code_dealloc(co);
-}
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-static void init_cpyprof(int native)
-{
-    // skip this if native should not be enabled
-    if (!native) {
-        vmp_native_disable();
-        return;
-    }
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = vmprof_eval;
-    _default_eval_loop = _PyEval_EvalFrameDefault;
-#else
-    if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx,
-                vmprof_eval, (void*)&_default_eval_loop) == 0) {
-    } else {
-        fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n");
-        // TODO dump the first few bytes and tell them to create an issue!
-        exit(-1);
-    }
-#endif
-    vmp_native_enable();
-}
-#endif
-
-void dump_native_symbols(int fileno)
-{
-    PyObject * mod = NULL;
-
-    mod = PyImport_ImportModuleNoBlock("vmprof");
-    if (mod == NULL)
-        goto error;
-
-    PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
-
-error:
-    Py_XDECREF(mod);
-}
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-static void disable_cpyprof(void)
-{
-    vmp_native_disable();
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = _PyEval_EvalFrameDefault;
-#else
-    if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) {
-        fprintf(stderr, "FATAL: could not remove trampoline\n");
-        exit(-1);
-    }
-#endif
-    dump_native_symbols(vmp_profile_fileno());
-}
-#endif
-
-
-static PyObject *enable_vmprof(PyObject* self, PyObject *args)
-{
-    int fd;
-    int memory = 0;
-    int lines = 0;
-    int native = 0;
-    double interval;
-    char *p_error;
-
-    if (!PyArg_ParseTuple(args, "id|iii", &fd, &interval, &memory, &lines, &native)) {
-        return NULL;
-    }
-    assert(fd >= 0 && "file descripter provided to vmprof must not" \
-                      " be less then zero.");
-
-    if (is_enabled) {
-        PyErr_SetString(PyExc_ValueError, "vmprof is already enabled");
-        return NULL;
-    }
-
-    vmp_profile_lines(lines);
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    init_cpyprof(native);
-#endif
-
-    if (!Original_code_dealloc) {
-        Original_code_dealloc = PyCode_Type.tp_dealloc;
-        PyCode_Type.tp_dealloc = &cpyprof_code_dealloc;
-    }
-
-    p_error = vmprof_init(fd, interval, memory, lines, "cpython", native);
-    if (p_error) {
-        PyErr_SetString(PyExc_ValueError, p_error);
-        return NULL;
-    }
-
-    if (vmprof_enable(memory) < 0) {
-        PyErr_SetFromErrno(PyExc_OSError);
-        return NULL;
-    }
-
-    is_enabled = 1;
-
-    Py_INCREF(Py_None);
-    return Py_None;
-}
-
-static PyObject *
-disable_vmprof(PyObject *module, PyObject *noarg)
-{
-    if (!is_enabled) {
-        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
-        return NULL;
-    }
-    is_enabled = 0;
-    vmprof_ignore_signals(1);
-    emit_all_code_objects();
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    disable_cpyprof();
-#endif
-
-    if (vmprof_disable() < 0) {
-        PyErr_SetFromErrno(PyExc_OSError);
-        return NULL;
-    }
-    if (PyErr_Occurred())
-        return NULL;
-    Py_INCREF(Py_None);
-    return Py_None;
-}
-
-static PyObject *
-write_all_code_objects(PyObject *module, PyObject *noargs)
-{
-    if (!is_enabled) {
-        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
-        return NULL;
-    }
-    emit_all_code_objects();
-    if (PyErr_Occurred())
-        return NULL;
-    Py_INCREF(Py_None);
-    return Py_None;
-}
-
-
-static PyObject *
-sample_stack_now(PyObject *module, PyObject *args)
-{
-    PyThreadState * tstate = NULL;
-    PyObject * list;
-    int i;
-    int entry_count;
-    void ** m;
-    void * routine_ip;
-
-    // stop any signal to occur
-    vmprof_ignore_signals(1);
-
-    list = PyList_New(0);
-    if (list == NULL) {
-        goto error;
-    }
-
-    tstate = PyGILState_GetThisThreadState();
-    m = (void**)malloc(SINGLE_BUF_SIZE);
-    if (m == NULL) {
-        PyErr_SetString(PyExc_MemoryError, "could not allocate buffer for stack trace");
-        vmprof_ignore_signals(0);
-        return NULL;
-    }
-    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, 0);
-
-    for (i = 0; i < entry_count; i++) {
-        routine_ip = m[i];
-        PyList_Append(list, PyLong_NEW((long)routine_ip));
-    }
-
-    free(m);
-
-    Py_INCREF(list);
-
-    vmprof_ignore_signals(0);
-    return list;
-error:
-    Py_DECREF(list);
-    Py_INCREF(Py_None);
-
-    vmprof_ignore_signals(0);
-    return Py_None;
-}
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-static PyObject *
-resolve_addr(PyObject *module, PyObject *args) {
-    long long addr;
-    PyObject * o_name = NULL;
-    PyObject * o_lineno = NULL;
-    PyObject * o_srcfile = NULL;
-    char name[128];
-    int lineno = 0;
-    char srcfile[256];
-
-    if (!PyArg_ParseTuple(args, "L", &addr)) {
-        return NULL;
-    }
-    name[0] = '\x00';
-    srcfile[0] = '-';
-    srcfile[1] = '\x00';
-    if (vmp_resolve_addr((void*)addr, name, 128, &lineno, srcfile, 256) != 0) {
-        goto error;
-    }
-
-    o_name = PyStr_NEW(name);
-    if (o_name == NULL) goto error;
-    o_lineno = PyLong_NEW(lineno);
-    if (o_lineno == NULL) goto error;
-    o_srcfile = PyStr_NEW(srcfile);
-    if (o_srcfile == NULL) goto error;
-    //
-    return PyTuple_Pack(3, o_name, o_lineno, o_srcfile);
-error:
-    Py_XDECREF(o_name);
-    Py_XDECREF(o_lineno);
-    Py_XDECREF(o_srcfile);
-
-    Py_INCREF(Py_None);
-    return Py_None;
-}
-#endif
-
-static PyMethodDef VMProfMethods[] = {
-    {"enable",  enable_vmprof, METH_VARARGS, "Enable profiling."},
-    {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."},
-    {"write_all_code_objects", write_all_code_objects, METH_NOARGS,
-     "Write eagerly all the IDs of code objects"},
-    {"sample_stack_now", sample_stack_now, METH_NOARGS, "Sample the stack now"},
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"},
-#endif
-    {NULL, NULL, 0, NULL}        /* Sentinel */
-};
-
-
-#if PY_MAJOR_VERSION >= 3
-static struct PyModuleDef VmprofModule = {
-    PyModuleDef_HEAD_INIT,
-    "_vmprof",
-    "",  // doc
-    -1,  // size
-    VMProfMethods
-};
-
-PyMODINIT_FUNC PyInit__vmprof(void)
-{
-    return PyModule_Create(&VmprofModule);
-}
-#else
-PyMODINIT_FUNC init_vmprof(void)
-{
-    Py_InitModule("_vmprof", VMProfMethods);
-}
-#endif
diff --git a/rpython/rlib/rvmprof/src/shared/src/_vmprof.h b/rpython/rlib/rvmprof/src/shared/src/_vmprof.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/src/_vmprof.h
+++ /dev/null
@@ -1,47 +0,0 @@
-#pragma once
-
-#include "vmprof.h"
-
-#ifdef VMPROF_WINDOWS
-#include "msiinttypes/inttypes.h"
-#include "msiinttypes/stdint.h"
-#else
-#include <inttypes.h>
-#include <stdint.h>
-#endif
-
-/**
- * This whole setup is very strange. There was just one C file called
- * _vmprof.c which included all *.h files to copy code. Unsure what
- * the goal was with this design, but I assume it just 'GREW'
- *
- * Thus I'm (plan_rich) slowly trying to separate this. *.h files
- * should not have complex implementations (all of them currently have them)
- */
-
-
-#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int))
-
-#define ROUTINE_IS_PYTHON(RIP) ((unsigned long long)RIP & 0x1) == 0
-#define ROUTINE_IS_C(RIP) ((unsigned long long)RIP & 0x1) == 1
-
-/* This returns the address of the code object
-   as the identifier.  The mapping from identifiers to string
-   representations of the code object is done elsewhere, namely:
-
-   * If the code object dies while vmprof is enabled,
-     PyCode_Type.tp_dealloc will emit it.  (We don't handle nicely
-     for now the case where several code objects are created and die
-     at the same memory address.)
-
-   * When _vmprof.disable() is called, then we look around the
-     process for code objects and emit all the ones that we can
-     find (which we hope is very close to 100% of them).
-*/
-#define CODE_ADDR_TO_UID(co)  (((intptr_t)(co)))
-
-#define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0
-
-PyObject* vmprof_eval(PyFrameObject *f, int throwflag);
-
-int vmp_write_all(const char *buf, size_t bufsize);
diff --git a/rpython/rlib/rvmprof/src/shared/src/compat.c b/rpython/rlib/rvmprof/src/shared/src/compat.c
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/src/compat.c
+++ /dev/null
@@ -1,137 +0,0 @@
-#include "compat.h"
-
-#include "_vmprof.h"
-
-#if VMPROF_WINDOWS
-#define WIN32_LEAN_AND_MEAN
-#include <Windows.h>
-#endif
-
-static int _vmp_profile_fileno = -1;
-
-int vmp_profile_fileno(void) {
-    return _vmp_profile_fileno;
-}
-void vmp_set_profile_fileno(int fileno) {
-    _vmp_profile_fileno = fileno;
-}
-
-#ifndef VMPROF_WINDOWS
-int vmp_write_all(const char *buf, size_t bufsize)
-{
-    ssize_t count;
-    if (_vmp_profile_fileno == -1) {
-        return -1;
-    }
-    while (bufsize > 0) {
-        count = write(_vmp_profile_fileno, buf, bufsize);
-        if (count <= 0)
-            return -1;   /* failed */
-        buf += count;
-        bufsize -= count;
-    }
-    return 0;
-}
-#endif
-
-int vmp_write_meta(const char * key, const char * value)
-{
-    char marker = MARKER_META;
-    long x = (long)strlen(key);
-    vmp_write_all(&marker, 1);
-    vmp_write_all((char*)&x, sizeof(long));
-    vmp_write_all(key, x);
-    x = (long)strlen(value);
-    vmp_write_all((char*)&x, sizeof(long));
-    vmp_write_all(value, x);
-    return 0;
-}
-
-/**
- * Write the time and zone now.
- */
-
-struct timezone_buf {
-    int64_t tv_sec;
-    int64_t tv_usec;
-};
-#define __SIZE (1+sizeof(struct timezone_buf)+8)
-
-#ifdef VMPROF_UNIX
-int vmp_write_time_now(int marker) {
-    char buffer[__SIZE];
-    struct timezone_buf buf;
-
-    (void)memset(&buffer, 0, __SIZE);
-
-    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
-           "marker must be either a trailer or time_n_zone!");
-
-    struct timeval tv;
-    time_t now;
-    struct tm tm;
-
-
-    /* copy over to the struct */
-    if (gettimeofday(&tv, NULL) != 0) {
-        return -1;
-    }
-    if (time(&now) == (time_t)-1) {
-        return -1;
-    }
-    if (localtime_r(&now, &tm) == NULL) {
-        return -1;
-    }
-    buf.tv_sec = tv.tv_sec;
-    buf.tv_usec = tv.tv_usec;
-    strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
-
-    buffer[0] = marker;
-    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
-    vmp_write_all(buffer, __SIZE);
-    return 0;
-}
-#endif
-
-#ifdef VMPROF_WINDOWS
-int vmp_write_time_now(int marker) {
-    char buffer[__SIZE];
-    struct timezone_buf buf;
-
-    /**
-     * http://stackoverflow.com/questions/10905892/equivalent-of-gettimeday-for-windows
-     */
-
-    // Note: some broken versions only have 8 trailing zero's, the correct
-    // epoch has 9 trailing zero's
-    static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL);
-
-    SYSTEMTIME  system_time;
-    FILETIME    file_time;
-    uint64_t    time;
-
-    (void)memset(&buffer, 0, __SIZE);
-
-    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
-           "marker must be either a trailer or time_n_zone!");
-
-
-    GetSystemTime( &system_time );
-    SystemTimeToFileTime( &system_time, &file_time );
-    time =  ((uint64_t)file_time.dwLowDateTime )      ;
-    time += ((uint64_t)file_time.dwHighDateTime) << 32;
-
-    buf.tv_sec = ((time - EPOCH) / 10000000L);
-    buf.tv_usec = (system_time.wMilliseconds * 1000);
-
-    // time zone not implemented on windows
-    memset(((char*)buffer)+__SIZE-8, 0, 8);
-    (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3);
-
-    buffer[0] = marker;
-    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
-    vmp_write_all(buffer, __SIZE);
-    return 0;
-}
-#endif
-#undef __SIZE
diff --git a/rpython/rlib/rvmprof/src/shared/src/compat.h b/rpython/rlib/rvmprof/src/shared/src/compat.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/src/compat.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#pragma once
-
-#include "vmprof.h"
-
-#ifndef RPYTHON_VMPROF
-#  if PY_MAJOR_VERSION >= 3
-      #define PyStr_AS_STRING PyBytes_AS_STRING
-      #define PyStr_GET_SIZE PyBytes_GET_SIZE
-      #define PyStr_NEW      PyUnicode_FromString
-      #define PyLong_NEW     PyLong_FromLong
-#  else
-      #define PyStr_AS_STRING PyString_AS_STRING
-      #define PyStr_GET_SIZE PyString_GET_SIZE
-      #define PyStr_NEW      PyString_FromString
-      #define PyLong_NEW     PyInt_FromLong
-#  endif
-#endif
-
-int vmp_write_all(const char *buf, size_t bufsize);
-int vmp_write_time_now(int marker);
-int vmp_write_meta(const char * key, const char * value);
-
-int vmp_profile_fileno(void);
-void vmp_set_profile_fileno(int fileno);
diff --git a/rpython/rlib/rvmprof/src/shared/src/libbacktrace/ChangeLog b/rpython/rlib/rvmprof/src/shared/src/libbacktrace/ChangeLog
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/src/libbacktrace/ChangeLog
+++ /dev/null
@@ -1,602 +0,0 @@
-2017-01-01  Jakub Jelinek  <jakub at redhat.com>
-
-	Update copyright years.
-
-2016-11-15  Matthias Klose  <doko at ubuntu.com>
-
-	* configure: Regenerate.
-
-2016-09-11  Carlos Liam  <carlos at aarzee.me>
-
-	* all: Remove meaningless trailing whitespace.
-
-2016-05-18  Uros Bizjak  <ubizjak at gmail.com>
-
-	PR target/71161
-	* elf.c (phdr_callback) [__i386__]: Add
-	__attribute__((__force_align_arg_pointer__)).
-
-2016-03-02  Maxim Ostapenko  <m.ostapenko at partner.samsung.com>
-
-	* elf.c (backtrace_initialize): Properly initialize elf_fileline_fn to
-	avoid possible crash.
-	(elf_add): Don't set *fileline_fn to elf_nodebug value in case of
-	missing debug info anymore.
-
-2016-02-06  John David Anglin  <danglin at gcc.gnu.org>
-
-	* mmap.c (MAP_FAILED): Define if not defined.
-
-2016-01-04  Jakub Jelinek  <jakub at redhat.com>
-
-	Update copyright years.
-
-2015-12-18  Andris Pavenis  <andris.pavenis at iki.fi>
-
-	* configure.ac: Specify that DJGPP do not have mmap
-	even when sys/mman.h exists.
-	* configure: Regenerate
-
-2015-12-09  John David Anglin  <danglin at gcc.gnu.org>
-
-	PR libgfortran/68115
-	* configure.ac: Set libbacktrace_cv_sys_sync to no on hppa*-*-hpux*.
-	* configure: Regenerate.
-	* elf.c (backtrace_initialize): Cast __sync_bool_compare_and_swap call
-	to void.
-
-2015-09-17  Ian Lance Taylor  <iant at google.com>
-
-	* posix.c (backtrace_open): Cast second argument of open() to int.
-
-2015-09-11  Ian Lance Taylor  <iant at google.com>
-
-	* Makefile.am (backtrace.lo): Depend on internal.h.
-	(sort.lo, stest.lo): Add explicit dependencies.
-	* Makefile.in: Rebuild.
-
-2015-09-09  Hans-Peter Nilsson  <hp at axis.com>
-
-	* backtrace.c: #include <sys/types.h>.
-
-2015-09-08  Ian Lance Taylor  <iant at google.com>
-
-	PR other/67457
-	* backtrace.c: #include "internal.h".
-	(struct backtrace_data): Add can_alloc field.
-	(unwind): If can_alloc is false, don't try to get file/line
-	information.
-	(backtrace_full): Set can_alloc field in bdata.
-	* alloc.c (backtrace_alloc): Don't call error_callback if it is
-	NULL.
-	* mmap.c (backtrace_alloc): Likewise.
-	* internal.h: Update comments for backtrace_alloc and
-	backtrace_free.
-
-2015-09-08  Ian Lance Taylor  <iant at google.com>
-
-	PR other/67457
-	* mmap.c (backtrace_alloc): Correct test for mmap failure.
-
-2015-08-31  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
-
-	* configure.ac: For spu-*-* targets, set have_fcntl to no.
-	* configure: Regenerate.
-
-2015-08-27  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
-
-	* configure.ac: Remove [disable-shared] argument to LT_INIT.
-	Remove setting PIC_FLAG when building as target library.
-	* configure: Regenerate.
-
-2015-08-26  Hans-Peter Nilsson  <hp at axis.com>
-
-	* configure.ac: Only compile with -fPIC if the target
-	supports it.
-	* configure: Regenerate.
-
-2015-08-24  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
-
-	* configure.ac: Set have_mmap to no on spu-*-* targets.
-	* configure: Regenerate.
-
-2015-08-13  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_function_entry): Add vec_inlined parameter.
-	Change all callers.
-
-2015-06-11  Martin Sebor  <msebor at redhat.com>
-
-	PR sanitizer/65479
-	* dwarf.c (struct line): Add new field idx.
-	(line_compare): Use it.
-	(add_line): Set it.
-	(read_line_info): Reset it.
-
-2015-05-29  Tristan Gingold  <gingold at adacore.com>
-
-	* pecoff.c: New file.
-	* Makefile.am (FORMAT_FILES): Add pecoff.c and dependencies.
-	* Makefile.in: Regenerate.
-	* filetype.awk: Detect pecoff.
-	* configure.ac: Define BACKTRACE_SUPPORTS_DATA on elf platforms.
-	Add pecoff.
-	* btest.c (test5): Test enabled only if BACKTRACE_SUPPORTS_DATA is
-	true.
-	* backtrace-supported.h.in (BACKTRACE_SUPPORTS_DATA): Define.
-	* configure: Regenerate.
-	* pecoff.c: New file.
-
-2015-05-13  Michael Haubenwallner  <michael.haubenwallner at ssi-schaefer.com>
-
-	* Makefile.in: Regenerated with automake-1.11.6.
-	* aclocal.m4: Likewise.
-	* configure: Likewise.
-
-2015-01-24  Matthias Klose  <doko at ubuntu.com>
-
-	* configure.ac: Move AM_ENABLE_MULTILIB before AC_PROG_CC.
-	* configure: Regenerate.
-
-2015-01-05  Jakub Jelinek  <jakub at redhat.com>
-
-	Update copyright years.
-
-2014-11-21  H.J. Lu  <hongjiu.lu at intel.com>
-
-	PR bootstrap/63784
-	* configure: Regenerated.
-
-2014-11-11  David Malcolm  <dmalcolm at redhat.com>
-
-	* ChangeLog.jit: New.
-
-2014-11-11  Francois-Xavier Coudert  <fxcoudert at gcc.gnu.org>
-
-	PR target/63610
-	* configure: Regenerate.
-
-2014-10-23  Ian Lance Taylor  <iant at google.com>
-
-	* internal.h (backtrace_atomic_load_pointer) [no atomic or sync]:
-	Fix to return void *.
-
-2014-05-08  Ian Lance Taylor  <iant at google.com>
-
-	* mmap.c (backtrace_free): If freeing a large aligned block of
-	memory, call munmap rather than holding onto it.
-	(backtrace_vector_grow): When growing a vector, double the number
-	of pages requested.  When releasing the old version of a grown
-	vector, pass the correct size to backtrace_free.
-
-2014-03-07  Ian Lance Taylor  <iant at google.com>
-
-	* sort.c (backtrace_qsort): Use middle element as pivot.
-
-2014-03-06  Ian Lance Taylor  <iant at google.com>
-
-	* sort.c: New file.
-	* stest.c: New file.
-	* internal.h (backtrace_qsort): Declare.
-	* dwarf.c (read_abbrevs): Call backtrace_qsort instead of qsort.
-	(read_line_info, read_function_entry): Likewise.
-	(read_function_info, build_dwarf_data): Likewise.
-	* elf.c (elf_initialize_syminfo): Likewise.
-	* Makefile.am (libbacktrace_la_SOURCES): Add sort.c.
-	(stest_SOURCES, stest_LDADD): Define.
-	(check_PROGRAMS): Add stest.
-
-2014-02-07  Misty De Meo  <misty at brew.sh>
-
-	PR target/58710
-	* configure.ac: Use AC_LINK_IFELSE in check for
-	_Unwind_GetIPInfo.
-	* configure: Regenerate.
-
-2014-01-02  Richard Sandiford  <rdsandiford at googlemail.com>
-
-	Update copyright years
-
-2013-12-06  Jakub Jelinek  <jakub at redhat.com>
-
-	* elf.c (ET_DYN): Undefine and define again.
-	(elf_add): Add exe argument, if true and ehdr.e_type is ET_DYN,
-	return early -1 without closing the descriptor.
-	(struct phdr_data): Add exe_descriptor.
-	(phdr_callback): If pd->exe_descriptor is not -1, for very first
-	call if dlpi_name is NULL just call elf_add with the exe_descriptor,
-	otherwise backtrace_close the exe_descriptor if not -1.  Adjust
-	call to elf_add.
-	(backtrace_initialize): Adjust call to elf_add.  If it returns
-	-1, set pd.exe_descriptor to descriptor, otherwise set it to -1.
-
-2013-12-05  Ian Lance Taylor  <iant at google.com>
-
-	* alloc.c (backtrace_vector_finish): Add error_callback and data
-	parameters.  Call backtrace_vector_release.  Return address base.
-	* mmap.c (backtrace_vector_finish): Add error_callback and data
-	parameters.  Return address base.
-	* dwarf.c (read_function_info): Get new address base from
-	backtrace_vector_finish.
-	* internal.h (backtrace_vector_finish): Update declaration.
-
-2013-11-27  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (find_address_ranges): New static function, broken out
-	of build_address_map.
-	(build_address_map): Call it.
-	* btest.c (check): Check for missing filename or function, rather
-	than crashing.
-	(f3): Check that enough frames were returned.
-
-2013-11-19  Jakub Jelinek  <jakub at redhat.com>
-
-	* backtrace.h (backtrace_syminfo_callback): Add symsize argument.
-	* elf.c (elf_syminfo): Pass 0 or sym->size to the callback as
-	last argument.
-	* btest.c (struct symdata): Add size field.
-	(callback_three): Add symsize argument.  Copy it to the data->size
-	field.
-	(f23): Set symdata.size to 0.
-	(test5): Likewise.  If sizeof (int) > 1, lookup address of
-	((uintptr_t) &global) + 1.  Verify symdata.val and symdata.size
-	values.
-
-	* atomic.c: Include sys/types.h.
-
-2013-11-18  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Check for support of __atomic extensions.
-	* internal.h: Declare or #define atomic functions for use in
-	backtrace code.
-	* atomic.c: New file.
-	* dwarf.c (dwarf_lookup_pc): Use atomic functions.
-	(dwarf_fileline, backtrace_dwarf_add): Likewise.
-	* elf.c (elf_add_syminfo_data, elf_syminfo): Likewise.
-	(backtrace_initialize): Likewise.
-	* fileline.c (fileline_initialize): Likewise.
-	* Makefile.am (libbacktrace_la_SOURCES): Add atomic.c.
-	* configure, config.h.in, Makefile.in: Rebuild.
-
-2013-11-18  Jakub Jelinek  <jakub at redhat.com>
-
-	* elf.c (SHN_UNDEF): Define.
-	(elf_initialize_syminfo): Add base_address argument.  Ignore symbols
-	with st_shndx == SHN_UNDEF.  Add base_address to address fields.
-	(elf_add): Adjust caller.
-
-	* elf.c (phdr_callback): Process info->dlpi_addr == 0 normally.
-
-2013-11-16  Ian Lance Taylor  <iant at google.com>
-
-	* backtrace.h (backtrace_create_state): Correct comment about
-	threading.
-
-2013-11-15  Ian Lance Taylor  <iant at google.com>
-
-	* backtrace.h (backtrace_syminfo): Update comment and parameter
-	name to take any address, not just a PC value.
-	* elf.c (STT_OBJECT): Define.
-	(elf_nosyms): Rename parameter pc to addr.
-	(elf_symbol_search): Rename local variable pc to addr.
-	(elf_initialize_syminfo): Add STT_OBJECT symbols to elf_symbols.
-	(elf_syminfo): Rename parameter pc to addr.
-	* btest.c (global): New global variable.
-	(test5): New test.
-	(main): Call test5.
-
-2013-10-17  Ian Lance Taylor  <iant at google.com>
-
-	* elf.c (elf_add): Don't get the wrong offsets if a debug section
-	is missing.
-
-2013-10-15  David Malcolm  <dmalcolm at redhat.com>
-
-	* configure.ac: Add --enable-host-shared, setting up
-	pre-existing PIC_FLAG variable within Makefile.am et al.
-	* configure: Regenerate.
-
-2013-09-20  Alan Modra  <amodra at gmail.com>
-
-	* configure: Regenerate.
-
-2013-07-23  Alexander Monakov  <amonakov at ispras.ru>
-
-	* elf.c (elf_syminfo): Loop over the elf_syminfo_data chain.
-
-2013-07-23  Alexander Monakov  <amonakov at ispras.ru>
-
-	* elf.c (backtrace_initialize): Pass elf_fileline_fn to
-	dl_iterate_phdr callbacks.
-
-2013-03-25  Ian Lance Taylor  <iant at google.com>
-
-	* alloc.c: #include <sys/types.h>.
-	* mmap.c: Likewise.
-
-2013-01-31  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_function_info): Permit fvec parameter to be NULL.
-	(dwarf_lookup_pc): Don't use ddata->fvec if threaded.
-
-2013-01-25  Jakub Jelinek  <jakub at redhat.com>
-
-	PR other/56076
-	* dwarf.c (read_line_header): Don't crash if DW_AT_comp_dir
-	attribute was not seen.
-
-2013-01-16  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (struct unit): Add filename and abs_filename fields.
-	(build_address_map): Set new fields when reading unit.
-	(dwarf_lookup_pc): If we don't find an entry in the line table,
-	just return the main file name.
-
-2013-01-14  Richard Sandiford  <rdsandiford at googlemail.com>
-
-	Update copyright years.
-
-2013-01-01  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54834
-	* Makefile.am (AM_CPPFLAGS): Remove -I ../gcc/include and -I
-	$(MULTIBUILDTOP)/../../gcc/include.
-	* Makefile.in: Rebuild.
-
-2013-01-01  Ian Lance Taylor  <iant at google.com>
-
-	PR other/55536
-	* mmap.c (backtrace_alloc): Don't call sync functions if not
-	threaded.
-	(backtrace_free): Likewise.
-
-2012-12-12  John David Anglin  <dave.anglin at nrc-cnrc.gc.ca>
-
-	* mmapio.c: Define MAP_FAILED if not defined.
-
-2012-12-11  Jakub Jelinek  <jakub at redhat.com>
-
-	PR bootstrap/54926
-	* Makefile.am (AM_CFLAGS): Remove -frandom-seed=$@.
-	* configure.ac: If --with-target-subdir, add -frandom-seed=$@
-	to EXTRA_FLAGS unconditionally, otherwise check whether the compiler
-	accepts it.
-	* Makefile.in: Regenerated.
-	* configure: Regenerated.
-
-2012-12-07  Jakub Jelinek  <jakub at redhat.com>
-
-	PR bootstrap/54926
-	* Makefile.am (AM_CFLAGS): Add -frandom-seed=$@.
-	* Makefile.in: Regenerated.
-
-2012-11-20  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_attribute): Always clear val.
-
-2012-11-13  Ian Lance Taylor  <iant at google.com>
-
-	PR other/55312
-	* configure.ac: Only add -Werror if building a target library.
-	* configure: Rebuild.
-
-2012-11-12  Ian Lance Taylor  <iant at google.com>
-	    Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
-	    Gerald Pfeifer  <gerald at pfeifer.com>
-
-	* configure.ac: Check for getexecname.
-	* fileline.c: #include <errno.h>.  Define getexecname if not
-	available.
-	(fileline_initialize): Try to find the executable in a few
-	different ways.
-	* print.c (error_callback): Only print the filename if it came
-	from the backtrace state.
-	* configure, config.h.in: Rebuild.
-
-2012-10-29  Ian Lance Taylor  <iant at google.com>
-
-	* mmap.c (backtrace_vector_release): Correct last patch: add
-	aligned, not size.
-
-2012-10-29  Ian Lance Taylor  <iant at google.com>
-
-	* mmap.c (backtrace_vector_release): Make sure freed block is
-	aligned on 8-byte boundary.
-
-2012-10-26  Ian Lance Taylor  <iant at google.com>
-
-	PR other/55087
-	* posix.c (backtrace_open): Add does_not_exist parameter.
-	* elf.c (phdr_callback): Do not warn if shared library could not
-	be opened.
-	* fileline.c (fileline_initialize): Update calls to
-	backtrace_open.
-	* internal.h (backtrace_open): Update declaration.
-
-2012-10-26  Jack Howarth  <howarth at bromo.med.uc.edu>
-
-	PR target/55061
-	* configure.ac: Check for _Unwind_GetIPInfo function declaration.
-	* configure: Regenerate.
-
-2012-10-24  Ian Lance Taylor  <iant at google.com>
-
-	PR target/55061
-	* configure.ac: Check whether -funwind-tables option works.
-	* configure: Rebuild.
-
-2012-10-11  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Do not use dl_iterate_phdr on Solaris 10.
-	* configure: Rebuild.
-
-2012-10-10  Ian Lance Taylor  <iant at google.com>
-
-	* elf.c: Rename all Elf typedefs to start with b_elf, and be all
-	lower case.
-
-2012-10-10  Hans-Peter Nilsson  <hp at bitrange.com>
-
-	* elf.c (elf_add_syminfo_data): Add casts to avoid warning.
-
-2012-10-09  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (dwarf_fileline): Add cast to avoid warning.
-	(backtrace_dwarf_add): Likewise.
-
-2012-10-09  Ian Lance Taylor  <iant at google.com>
-
-	Add support for tracing through shared libraries.
-	* configure.ac: Check for link.h and dl_iterate_phdr.
-	* elf.c: #include <link.h> if system has dl_iterate_phdr.  #undef
-	ELF macros before #defining them.
-	(dl_phdr_info, dl_iterate_phdr): Define if system does not have
-	dl_iterate_phdr.
-	(struct elf_syminfo_data): Add next field.
-	(elf_initialize_syminfo): Initialize next field.
-	(elf_add_syminfo_data): New static function.
-	(elf_add): New static function, broken out of
-	backtrace_initialize.  Call backtrace_dwarf_add instead of
-	backtrace_dwarf_initialize.
-	(struct phdr_data): Define.
-	(phdr_callback): New static function.
-	(backtrace_initialize): Call elf_add.
-	* dwarf.c (struct dwarf_data): Add next and base_address fields.
-	(add_unit_addr): Add base_address parameter.  Change all callers.
-	(add_unit_ranges, build_address_map): Likewise.
-	(add_line): Add ddata parameter.  Change all callers.
-	(read_line_program, add_function_range): Likewise.
-	(dwarf_lookup_pc): New static function, broken out of
-	dwarf_fileline.
-	(dwarf_fileline): Call dwarf_lookup_pc.
-	(build_dwarf_data): New static function.
-	(backtrace_dwarf_add): New function.
-	(backtrace_dwarf_initialize): Remove.
-	* internal.h (backtrace_dwarf_initialize): Don't declare.
-	(backtrace_dwarf_add): Declare.
-	* configure, config.h.in: Rebuild.
-
-2012-10-04  Gerald Pfeifer  <gerald at pfeifer.com>
-
-	* btest.c (f23): Avoid uninitialized variable warning.
-
-2012-10-04  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c: If the system header files do not declare strnlen,
-	provide our own version.
-
-2012-10-03  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c (read_uleb128): Fix overflow test.
-	(read_sleb128): Likewise.
-	(build_address_map): Don't change unit_buf.start.
-
-2012-10-02  Uros Bizjak  <ubizjak at gmail.com>
-
-	PR other/54761
-	* configure.ac (EXTRA_FLAGS): New.
-	* Makefile.am (AM_FLAGS): Add $(EXTRA_FLAGS).
-	* configure, Makefile.in: Regenerate.
-
-2012-09-29  Ian Lance Taylor  <iant at google.com>
-
-	PR other/54749
-	* fileline.c (fileline_initialize): Pass errnum as -1 when
-	reporting that we could not read executable information after a
-	previous failure.
-
-2012-09-27  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54732
-	* configure.ac: Add no-dependencies to AM_INIT_AUTOMAKE.
-	* Makefile.am: Add dependencies for all objects.
-	* configure, aclocal.m4, Makefile.in: Rebuild.
-
-2012-09-27  Ian Lance Taylor  <iant at google.com>
-
-	PR other/54726
-	* elf.c (backtrace_initialize): Set *fileln_fn, not
-	state->fileln_fn.
-
-2012-09-19  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Only use GCC_CHECK_UNWIND_GETIPINFO when compiled
-	as a target library.
-	* configure: Rebuild.
-
-2012-09-19  Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
-	    Ian Lance Taylor  <iant at google.com>
-
-        * configure.ac (GCC_HEADER_STDINT): Invoke.
-        * backtrace.h: If we can't find <stdint.h>, use "gstdint.h".
-        * btest.c: Don't include <stdint.h>.
-        * dwarf.c: Likewise.
-        * configure, aclocal.m4, Makefile.in, config.h.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54623
-	* Makefile.am (AM_CPPFLAGS): Define.
-	(AM_CFLAGS): Remove -I options.
-	* Makefile.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* posix.c (O_BINARY): Define if not defined.
-	(backtrace_open): Pass O_BINARY to open.  Only call fcntl if
-	HAVE_FCNTL is defined.
-	* configure.ac: Test for the fcntl function.
-	* configure, config.h.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* btest.c (test1, test2, test3, test4): Add the unused attribute.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* dwarf.c: Correct test of HAVE_DECL_STRNLEN.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Add AC_USE_SYSTEM_EXTENSIONS.
-	* mmapio.c: Don't define _GNU_SOURCE.
-	* configure, config.h.in: Rebuild.
-
-2012-09-18  Ian Lance Taylor  <iant at google.com>
-
-	* configure.ac: Check whether strnlen is declared.
-	* dwarf.c: Declare strnlen if not declared.
-	* configure, config.h.in: Rebuild.
-
-2012-09-18  Rainer Orth  <ro at CeBiTec.Uni-Bielefeld.DE>
-
-	* fileline.c: Include <stdlib.h>.
-	* mmap.c: Likewise.
-
-2012-09-17  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54611
-	* nounwind.c (backtrace_full): Rename from backtrace.  Add state
-	parameter.
-
-2012-09-17  Gerald Pfeifer  <gerald at pfeifer.com>
-
-	PR bootstrap/54611
-	* nounwind.c (backtrace_simple): Add state parameter.
-
-2012-09-17  Ian Lance Taylor  <iant at google.com>
-
-	PR bootstrap/54609
-	* unknown.c (unknown_fileline): Add state parameter, remove
-	fileline_data parameter, name error_callback parameter.
-	(backtrace_initialize): Add state parameter.
-
-2012-09-17  Ian Lance Taylor  <iant at google.com>
-
-	* Initial implementation.
-

-Copyright (C) 2012-2017 Free Software Foundation, Inc.
-
-Copying and distribution of this file, with or without modification,
-are permitted in any medium without royalty provided the copyright
-notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/shared/src/libbacktrace/ChangeLog.jit b/rpython/rlib/rvmprof/src/shared/src/libbacktrace/ChangeLog.jit
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/src/libbacktrace/ChangeLog.jit
+++ /dev/null
@@ -1,14 +0,0 @@
-2014-09-24  David Malcolm  <dmalcolm at redhat.com>
-
-	* ChangeLog.jit: Add copyright footer.
-
-2013-10-03  David Malcolm  <dmalcolm at redhat.com>
-
-	* configure.ac: Add --enable-host-shared.
-	* configure: Regenerate.
-

-Copyright (C) 2013-2014 Free Software Foundation, Inc.
-
-Copying and distribution of this file, with or without modification,
-are permitted in any medium without royalty provided the copyright
-notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/shared/src/libbacktrace/Makefile b/rpython/rlib/rvmprof/src/shared/src/libbacktrace/Makefile
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/src/libbacktrace/Makefile
+++ /dev/null
@@ -1,770 +0,0 @@
-# Makefile.in generated by automake 1.11.6 from Makefile.am.
-# Makefile.  Generated from Makefile.in by configure.
-
-# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
-# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-# Foundation, Inc.
-# This Makefile.in is free software; the Free Software Foundation
-# gives unlimited permission to copy and/or distribute it,
-# with or without modifications, as long as this notice is preserved.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
-# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE.
-
-
-
-# Makefile.am -- Backtrace Makefile.
-# Copyright (C) 2012-2016 Free Software Foundation, Inc.
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-
-#     (1) Redistributions of source code must retain the above copyright
-#     notice, this list of conditions and the following disclaimer.
-
-#     (2) Redistributions in binary form must reproduce the above copyright
-#     notice, this list of conditions and the following disclaimer in
-#     the documentation and/or other materials provided with the
-#     distribution.
-
-#     (3) The name of the author may not be used to
-#     endorse or promote products derived from this software without
-#     specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-
-am__make_dryrun = \
-  { \
-    am__dry=no; \
-    case $$MAKEFLAGS in \
-      *\\[\ \	]*) \
-        echo 'am--echo: ; @echo "AM"  OK' | $(MAKE) -f - 2>/dev/null \
-          | grep '^AM OK$$' >/dev/null || am__dry=yes;; \
-      *) \
-        for am__flg in $$MAKEFLAGS; do \
-          case $$am__flg in \
-            *=*|--*) ;; \
-            *n*) am__dry=yes; break;; \
-          esac; \
-        done;; \
-    esac; \
-    test $$am__dry = yes; \
-  }
-pkgdatadir = $(datadir)/libbacktrace
-pkgincludedir = $(includedir)/libbacktrace
-pkglibdir = $(libdir)/libbacktrace
-pkglibexecdir = $(libexecdir)/libbacktrace
-am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
-install_sh_DATA = $(install_sh) -c -m 644
-install_sh_PROGRAM = $(install_sh) -c
-install_sh_SCRIPT = $(install_sh) -c
-INSTALL_HEADER = $(INSTALL_DATA)
-transform = $(program_transform_name)
-NORMAL_INSTALL = :
-PRE_INSTALL = :
-POST_INSTALL = :
-NORMAL_UNINSTALL = :
-PRE_UNINSTALL = :
-POST_UNINSTALL = :
-build_triplet = x86_64-pc-linux-gnu
-host_triplet = x86_64-pc-linux-gnu
-target_triplet = x86_64-pc-linux-gnu
-check_PROGRAMS = $(am__EXEEXT_1)
-am__append_1 = btest stest
-subdir = .
-DIST_COMMON = README ChangeLog $(srcdir)/Makefile.in \
-	$(srcdir)/Makefile.am $(top_srcdir)/configure \
-	$(am__configure_deps) $(srcdir)/config.h.in \
-	$(srcdir)/../mkinstalldirs $(srcdir)/backtrace-supported.h.in
-ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/../config/lead-dot.m4 \
-	$(top_srcdir)/../config/multi.m4 \
-	$(top_srcdir)/../config/override.m4 \
-	$(top_srcdir)/../config/stdint.m4 \
-	$(top_srcdir)/../config/unwind_ipinfo.m4 \
-	$(top_srcdir)/../config/warnings.m4 \
-	$(top_srcdir)/../libtool.m4 $(top_srcdir)/../ltoptions.m4 \
-	$(top_srcdir)/../ltsugar.m4 $(top_srcdir)/../ltversion.m4 \
-	$(top_srcdir)/../lt~obsolete.m4 $(top_srcdir)/configure.ac
-am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
-	$(ACLOCAL_M4)
-am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
- configure.lineno config.status.lineno
-mkinstalldirs = $(SHELL) $(top_srcdir)/../mkinstalldirs
-CONFIG_HEADER = config.h
-CONFIG_CLEAN_FILES = backtrace-supported.h
-CONFIG_CLEAN_VPATH_FILES =
-LTLIBRARIES = $(noinst_LTLIBRARIES)
-am__DEPENDENCIES_1 =
-am_libbacktrace_la_OBJECTS = atomic.lo dwarf.lo fileline.lo posix.lo \
-	print.lo sort.lo state.lo
-libbacktrace_la_OBJECTS = $(am_libbacktrace_la_OBJECTS)
-am__EXEEXT_1 = btest$(EXEEXT) stest$(EXEEXT)
-am_btest_OBJECTS = btest-btest.$(OBJEXT)
-btest_OBJECTS = $(am_btest_OBJECTS)
-btest_DEPENDENCIES = libbacktrace.la
-btest_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=link $(CCLD) $(btest_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
-	$(LDFLAGS) -o $@
-am_stest_OBJECTS = stest.$(OBJEXT)
-stest_OBJECTS = $(am_stest_OBJECTS)
-stest_DEPENDENCIES = libbacktrace.la
-DEFAULT_INCLUDES = -I.
-depcomp =
-am__depfiles_maybe =
-COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
-	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
-LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
-	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
-CCLD = $(CC)
-LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
-	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
-	$(LDFLAGS) -o $@
-SOURCES = $(libbacktrace_la_SOURCES) $(EXTRA_libbacktrace_la_SOURCES) \
-	$(btest_SOURCES) $(stest_SOURCES)
-MULTISRCTOP = 
-MULTIBUILDTOP = 
-MULTIDIRS = 
-MULTISUBDIR = 
-MULTIDO = true
-MULTICLEAN = true
-am__can_run_installinfo = \
-  case $$AM_UPDATE_INFO_DIR in \
-    n|no|NO) false;; \
-    *) (install-info --version) >/dev/null 2>&1;; \
-  esac
-ETAGS = etags
-CTAGS = ctags
-am__tty_colors = \
-red=; grn=; lgn=; blu=; std=
-ACLOCAL = ${SHELL} /home/rich/src/gcc/missing --run aclocal-1.11
-ALLOC_FILE = mmap.lo
-AMTAR = $${TAR-tar}
-AR = ar
-AUTOCONF = ${SHELL} /home/rich/src/gcc/missing --run autoconf
-AUTOHEADER = ${SHELL} /home/rich/src/gcc/missing --run autoheader
-AUTOMAKE = ${SHELL} /home/rich/src/gcc/missing --run automake-1.11
-AWK = gawk
-BACKTRACE_FILE = backtrace.lo simple.lo
-BACKTRACE_SUPPORTED = 1
-BACKTRACE_SUPPORTS_DATA = 1
-BACKTRACE_SUPPORTS_THREADS = 1
-BACKTRACE_USES_MALLOC = 0
-CC = gcc
-CFLAGS = -g -O2
-CPP = gcc -E
-CPPFLAGS = 
-CYGPATH_W = echo
-DEFS = -DHAVE_CONFIG_H
-DSYMUTIL = 
-DUMPBIN = 
-ECHO_C = 
-ECHO_N = -n
-ECHO_T = 
-EGREP = /usr/bin/grep -E
-EXEEXT = 
-EXTRA_FLAGS = -funwind-tables -frandom-seed=$@
-FGREP = /usr/bin/grep -F
-FORMAT_FILE = elf.lo
-GREP = /usr/bin/grep
-INSTALL = /usr/bin/install -c
-INSTALL_DATA = ${INSTALL} -m 644
-INSTALL_PROGRAM = ${INSTALL}
-INSTALL_SCRIPT = ${INSTALL}
-INSTALL_STRIP_PROGRAM = $(install_sh) -c -s
-LD = /usr/bin/ld -m elf_x86_64
-LDFLAGS = 
-LIBOBJS = 
-LIBS = 
-LIBTOOL = $(SHELL) $(top_builddir)/libtool
-LIPO = 
-LN_S = ln -s
-LTLIBOBJS = 
-MAINT = #
-MAKEINFO = ${SHELL} /home/rich/src/gcc/missing --run makeinfo
-MKDIR_P = /usr/bin/mkdir -p
-NM = /usr/bin/nm -B
-NMEDIT = 
-OBJDUMP = objdump
-OBJEXT = o
-OTOOL = 
-OTOOL64 = 
-PACKAGE = libbacktrace
-PACKAGE_BUGREPORT = 
-PACKAGE_NAME = package-unused
-PACKAGE_STRING = package-unused version-unused
-PACKAGE_TARNAME = libbacktrace
-PACKAGE_URL = 
-PACKAGE_VERSION = version-unused
-PATH_SEPARATOR = :
-PIC_FLAG = 
-RANLIB = ranlib
-SED = /usr/bin/sed
-SET_MAKE = 
-SHELL = /bin/sh
-STRIP = strip
-VERSION = version-unused
-VIEW_FILE = mmapio.lo
-WARN_FLAGS = -W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual
-abs_builddir = /home/rich/src/gcc/libbacktrace
-abs_srcdir = /home/rich/src/gcc/libbacktrace
-abs_top_builddir = /home/rich/src/gcc/libbacktrace
-abs_top_srcdir = /home/rich/src/gcc/libbacktrace
-ac_ct_CC = gcc
-ac_ct_DUMPBIN = 
-am__leading_dot = .
-am__tar = $${TAR-tar} chof - "$$tardir"
-am__untar = $${TAR-tar} xf -
-bindir = ${exec_prefix}/bin
-build = x86_64-pc-linux-gnu
-build_alias = 
-build_cpu = x86_64
-build_os = linux-gnu
-build_vendor = pc
-builddir = .
-datadir = ${datarootdir}
-datarootdir = ${prefix}/share
-docdir = ${datarootdir}/doc/${PACKAGE_TARNAME}
-dvidir = ${docdir}
-exec_prefix = ${prefix}
-host = x86_64-pc-linux-gnu
-host_alias = 
-host_cpu = x86_64
-host_os = linux-gnu
-host_vendor = pc
-htmldir = ${docdir}
-includedir = ${prefix}/include
-infodir = ${datarootdir}/info
-install_sh = ${SHELL} /home/rich/src/gcc/install-sh
-libdir = ${exec_prefix}/lib
-libexecdir = ${exec_prefix}/libexec
-libtool_VERSION = 1:0:0
-localedir = ${datarootdir}/locale
-localstatedir = ${prefix}/var
-mandir = ${datarootdir}/man
-mkdir_p = /usr/bin/mkdir -p
-multi_basedir = 
-oldincludedir = /usr/include
-pdfdir = ${docdir}
-prefix = /usr/local
-program_transform_name = s,x,x,
-psdir = ${docdir}
-sbindir = ${exec_prefix}/sbin
-sharedstatedir = ${prefix}/com
-srcdir = .
-sysconfdir = ${prefix}/etc
-target = x86_64-pc-linux-gnu
-target_alias = 
-target_cpu = x86_64
-target_os = linux-gnu
-target_vendor = pc
-top_build_prefix = 
-top_builddir = .
-top_srcdir = .
-ACLOCAL_AMFLAGS = -I .. -I ../config
-AM_CPPFLAGS = -I $(top_srcdir)/../include -I $(top_srcdir)/../libgcc \
-	-I ../libgcc
-
-AM_CFLAGS = $(EXTRA_FLAGS) $(WARN_FLAGS) $(PIC_FLAG)
-noinst_LTLIBRARIES = libbacktrace.la
-libbacktrace_la_SOURCES = \
-	backtrace.h \
-	atomic.c \
-	dwarf.c \
-	fileline.c \
-	internal.h \
-	posix.c \
-	print.c \
-	sort.c \
-	state.c
-
-BACKTRACE_FILES = \
-	backtrace.c \
-	simple.c \
-	nounwind.c
-
-FORMAT_FILES = \
-	elf.c \
-	pecoff.c \
-	unknown.c
-
-VIEW_FILES = \
-	read.c \
-	mmapio.c
-
-ALLOC_FILES = \
-	alloc.c \
-	mmap.c
-
-EXTRA_libbacktrace_la_SOURCES = \
-	$(BACKTRACE_FILES) \
-	$(FORMAT_FILES) \
-	$(VIEW_FILES) \
-	$(ALLOC_FILES)
-
-libbacktrace_la_LIBADD = \
-	$(BACKTRACE_FILE) \
-	$(FORMAT_FILE) \
-	$(VIEW_FILE) \
-	$(ALLOC_FILE)
-
-libbacktrace_la_DEPENDENCIES = $(libbacktrace_la_LIBADD)
-TESTS = $(check_PROGRAMS)
-btest_SOURCES = btest.c
-btest_CFLAGS = $(AM_CFLAGS) -g -O
-btest_LDADD = libbacktrace.la
-stest_SOURCES = stest.c
-stest_LDADD = libbacktrace.la
-
-# We can't use automake's automatic dependency tracking, because it
-# breaks when using bootstrap-lean.  Automatic dependency tracking
-# with GCC bootstrap will cause some of the objects to depend on
-# header files in prev-gcc/include, e.g., stddef.h and stdarg.h.  When
-# using bootstrap-lean, prev-gcc is removed after each stage.  When
-# running "make install", those header files will be gone, causing the
-# library to be rebuilt at install time.  That may not succeed.
-
-# These manual dependencies do not include dependencies on unwind.h,
-# even though that is part of GCC, because where to find it depends on
-# whether we are being built as a host library or a target library.
-INCDIR = $(top_srcdir)/../include
-all: config.h
-	$(MAKE) $(AM_MAKEFLAGS) all-am
-
-.SUFFIXES:
-.SUFFIXES: .c .lo .o .obj
-am--refresh: Makefile
-	@:
-$(srcdir)/Makefile.in: # $(srcdir)/Makefile.am  $(am__configure_deps)
-	@for dep in $?; do \
-	  case '$(am__configure_deps)' in \
-	    *$$dep*) \
-	      echo ' cd $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps'; \
-	      $(am__cd) $(srcdir) && $(AUTOMAKE) --foreign --ignore-deps \
-		&& exit 0; \
-	      exit 1;; \
-	  esac; \
-	done; \
-	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign --ignore-deps Makefile'; \
-	$(am__cd) $(top_srcdir) && \
-	  $(AUTOMAKE) --foreign --ignore-deps Makefile
-.PRECIOUS: Makefile
-Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
-	@case '$?' in \
-	  *config.status*) \
-	    echo ' $(SHELL) ./config.status'; \
-	    $(SHELL) ./config.status;; \
-	  *) \
-	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \
-	    cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \
-	esac;
-
-$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
-	$(SHELL) ./config.status --recheck
-
-$(top_srcdir)/configure: # $(am__configure_deps)
-	$(am__cd) $(srcdir) && $(AUTOCONF)
-$(ACLOCAL_M4): # $(am__aclocal_m4_deps)
-	$(am__cd) $(srcdir) && $(ACLOCAL) $(ACLOCAL_AMFLAGS)
-$(am__aclocal_m4_deps):
-
-config.h: stamp-h1
-	@if test ! -f $@; then rm -f stamp-h1; else :; fi
-	@if test ! -f $@; then $(MAKE) $(AM_MAKEFLAGS) stamp-h1; else :; fi
-
-stamp-h1: $(srcdir)/config.h.in $(top_builddir)/config.status
-	@rm -f stamp-h1
-	cd $(top_builddir) && $(SHELL) ./config.status config.h
-$(srcdir)/config.h.in: # $(am__configure_deps) 
-	($(am__cd) $(top_srcdir) && $(AUTOHEADER))
-	rm -f stamp-h1
-	touch $@
-
-distclean-hdr:
-	-rm -f config.h stamp-h1
-backtrace-supported.h: $(top_builddir)/config.status $(srcdir)/backtrace-supported.h.in
-	cd $(top_builddir) && $(SHELL) ./config.status $@
-
-clean-noinstLTLIBRARIES:
-	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
-	@list='$(noinst_LTLIBRARIES)'; for p in $$list; do \
-	  dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
-	  test "$$dir" != "$$p" || dir=.; \
-	  echo "rm -f \"$${dir}/so_locations\""; \
-	  rm -f "$${dir}/so_locations"; \
-	done
-libbacktrace.la: $(libbacktrace_la_OBJECTS) $(libbacktrace_la_DEPENDENCIES) $(EXTRA_libbacktrace_la_DEPENDENCIES) 
-	$(LINK)  $(libbacktrace_la_OBJECTS) $(libbacktrace_la_LIBADD) $(LIBS)
-
-clean-checkPROGRAMS:
-	@list='$(check_PROGRAMS)'; test -n "$$list" || exit 0; \
-	echo " rm -f" $$list; \
-	rm -f $$list || exit $$?; \
-	test -n "$(EXEEXT)" || exit 0; \
-	list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
-	echo " rm -f" $$list; \
-	rm -f $$list
-btest$(EXEEXT): $(btest_OBJECTS) $(btest_DEPENDENCIES) $(EXTRA_btest_DEPENDENCIES) 
-	@rm -f btest$(EXEEXT)
-	$(btest_LINK) $(btest_OBJECTS) $(btest_LDADD) $(LIBS)
-stest$(EXEEXT): $(stest_OBJECTS) $(stest_DEPENDENCIES) $(EXTRA_stest_DEPENDENCIES) 
-	@rm -f stest$(EXEEXT)
-	$(LINK) $(stest_OBJECTS) $(stest_LDADD) $(LIBS)
-
-mostlyclean-compile:
-	-rm -f *.$(OBJEXT)
-
-distclean-compile:
-	-rm -f *.tab.c
-
-.c.o:
-	$(COMPILE) -c $<
-
-.c.obj:
-	$(COMPILE) -c `$(CYGPATH_W) '$<'`
-
-.c.lo:
-	$(LTCOMPILE) -c -o $@ $<
-
-btest-btest.o: btest.c
-	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(btest_CFLAGS) $(CFLAGS) -c -o btest-btest.o `test -f 'btest.c' || echo '$(srcdir)/'`btest.c
-
-btest-btest.obj: btest.c
-	$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(btest_CFLAGS) $(CFLAGS) -c -o btest-btest.obj `if test -f 'btest.c'; then $(CYGPATH_W) 'btest.c'; else $(CYGPATH_W) '$(srcdir)/btest.c'; fi`
-
-mostlyclean-libtool:
-	-rm -f *.lo
-
-clean-libtool:
-	-rm -rf .libs _libs
-
-distclean-libtool:
-	-rm -f libtool config.lt
-
-# GNU Make needs to see an explicit $(MAKE) variable in the command it
-# runs to enable its job server during parallel builds.  Hence the
-# comments below.
-all-multi:
-	$(MULTIDO) $(AM_MAKEFLAGS) DO=all multi-do # $(MAKE)
-install-multi:
-	$(MULTIDO) $(AM_MAKEFLAGS) DO=install multi-do # $(MAKE)
-
-mostlyclean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=mostlyclean multi-clean # $(MAKE)
-clean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=clean multi-clean # $(MAKE)
-distclean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=distclean multi-clean # $(MAKE)
-maintainer-clean-multi:
-	$(MULTICLEAN) $(AM_MAKEFLAGS) DO=maintainer-clean multi-clean # $(MAKE)
-
-ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
-	list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
-	unique=`for i in $$list; do \
-	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
-	  done | \
-	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
-	      END { if (nonempty) { for (i in files) print i; }; }'`; \
-	mkid -fID $$unique
-tags: TAGS
-
-TAGS:  $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
-		$(TAGS_FILES) $(LISP)
-	set x; \
-	here=`pwd`; \
-	list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
-	unique=`for i in $$list; do \
-	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
-	  done | \
-	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
-	      END { if (nonempty) { for (i in files) print i; }; }'`; \
-	shift; \
-	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
-	  test -n "$$unique" || unique=$$empty_fix; \
-	  if test $$# -gt 0; then \
-	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
-	      "$$@" $$unique; \
-	  else \
-	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
-	      $$unique; \
-	  fi; \
-	fi
-ctags: CTAGS
-CTAGS:  $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
-		$(TAGS_FILES) $(LISP)
-	list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
-	unique=`for i in $$list; do \
-	    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
-	  done | \
-	  $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
-	      END { if (nonempty) { for (i in files) print i; }; }'`; \
-	test -z "$(CTAGS_ARGS)$$unique" \
-	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
-	     $$unique
-
-GTAGS:
-	here=`$(am__cd) $(top_builddir) && pwd` \
-	  && $(am__cd) $(top_srcdir) \
-	  && gtags -i $(GTAGS_ARGS) "$$here"
-
-distclean-tags:
-	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
-
-check-TESTS: $(TESTS)
-	@failed=0; all=0; xfail=0; xpass=0; skip=0; \
-	srcdir=$(srcdir); export srcdir; \
-	list=' $(TESTS) '; \
-	$(am__tty_colors); \
-	if test -n "$$list"; then \
-	  for tst in $$list; do \
-	    if test -f ./$$tst; then dir=./; \
-	    elif test -f $$tst; then dir=; \
-	    else dir="$(srcdir)/"; fi; \
-	    if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
-	      all=`expr $$all + 1`; \
-	      case " $(XFAIL_TESTS) " in \
-	      *[\ \	]$$tst[\ \	]*) \
-		xpass=`expr $$xpass + 1`; \
-		failed=`expr $$failed + 1`; \
-		col=$$red; res=XPASS; \
-	      ;; \
-	      *) \
-		col=$$grn; res=PASS; \
-	      ;; \
-	      esac; \
-	    elif test $$? -ne 77; then \
-	      all=`expr $$all + 1`; \
-	      case " $(XFAIL_TESTS) " in \
-	      *[\ \	]$$tst[\ \	]*) \
-		xfail=`expr $$xfail + 1`; \
-		col=$$lgn; res=XFAIL; \
-	      ;; \
-	      *) \
-		failed=`expr $$failed + 1`; \
-		col=$$red; res=FAIL; \
-	      ;; \
-	      esac; \
-	    else \
-	      skip=`expr $$skip + 1`; \
-	      col=$$blu; res=SKIP; \
-	    fi; \
-	    echo "$${col}$$res$${std}: $$tst"; \
-	  done; \
-	  if test "$$all" -eq 1; then \
-	    tests="test"; \
-	    All=""; \
-	  else \
-	    tests="tests"; \
-	    All="All "; \
-	  fi; \
-	  if test "$$failed" -eq 0; then \
-	    if test "$$xfail" -eq 0; then \
-	      banner="$$All$$all $$tests passed"; \
-	    else \
-	      if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \
-	      banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \
-	    fi; \
-	  else \
-	    if test "$$xpass" -eq 0; then \
-	      banner="$$failed of $$all $$tests failed"; \
-	    else \
-	      if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \
-	      banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \
-	    fi; \
-	  fi; \
-	  dashes="$$banner"; \
-	  skipped=""; \
-	  if test "$$skip" -ne 0; then \
-	    if test "$$skip" -eq 1; then \
-	      skipped="($$skip test was not run)"; \
-	    else \
-	      skipped="($$skip tests were not run)"; \
-	    fi; \
-	    test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \
-	      dashes="$$skipped"; \
-	  fi; \
-	  report=""; \
-	  if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \
-	    report="Please report to $(PACKAGE_BUGREPORT)"; \
-	    test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \
-	      dashes="$$report"; \
-	  fi; \
-	  dashes=`echo "$$dashes" | sed s/./=/g`; \
-	  if test "$$failed" -eq 0; then \
-	    col="$$grn"; \
-	  else \
-	    col="$$red"; \
-	  fi; \
-	  echo "$${col}$$dashes$${std}"; \
-	  echo "$${col}$$banner$${std}"; \
-	  test -z "$$skipped" || echo "$${col}$$skipped$${std}"; \
-	  test -z "$$report" || echo "$${col}$$report$${std}"; \
-	  echo "$${col}$$dashes$${std}"; \
-	  test "$$failed" -eq 0; \
-	else :; fi
-check-am: all-am
-	$(MAKE) $(AM_MAKEFLAGS) $(check_PROGRAMS)
-	$(MAKE) $(AM_MAKEFLAGS) check-TESTS
-check: check-am
-all-am: Makefile $(LTLIBRARIES) all-multi config.h
-installdirs:
-install: install-am
-install-exec: install-exec-am
-install-data: install-data-am

From pypy.commits at gmail.com  Mon Feb  6 12:43:40 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 06 Feb 2017 09:43:40 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: moved more files
Message-ID: <5898b5cc.cfc41c0a.9d5d2.9e5d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89975:bd9d1a5a539b
Date: 2017-02-06 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/bd9d1a5a539b/

Log:	moved more files

diff too long, truncating to 2000 out of 115559 lines

diff --git a/rpython/rlib/rvmprof/src/_vmprof.h b/rpython/rlib/rvmprof/src/_vmprof.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/_vmprof.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#pragma once
-
-#include <Python.h>
-#include <frameobject.h>
-
-#ifdef VMPROF_WINDOWS
-#include "msiinttypes/inttypes.h"
-#include "msiinttypes/stdint.h"
-#else
-#include <inttypes.h>
-#include <stdint.h>
-#endif
-
-/**
- * This whole setup is very strange. There was just one C file called
- * _vmprof.c which included all *.h files to copy code. Unsure what
- * the goal was with this design, but I assume it just 'GREW'
- *
- * Thus I'm (plan_rich) slowly trying to separate this. *.h files
- * should not have complex implementations (all of them currently have them)
- */
-
-
-#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int))
-
-#define ROUTINE_IS_PYTHON(RIP) ((unsigned long long)RIP & 0x1) == 0
-#define ROUTINE_IS_C(RIP) ((unsigned long long)RIP & 0x1) == 1
-
-typedef uint64_t ptr_t;
-
-/* This returns the address of the code object
-   as the identifier.  The mapping from identifiers to string
-   representations of the code object is done elsewhere, namely:
-
-   * If the code object dies while vmprof is enabled,
-     PyCode_Type.tp_dealloc will emit it.  (We don't handle nicely
-     for now the case where several code objects are created and die
-     at the same memory address.)
-
-   * When _vmprof.disable() is called, then we look around the
-     process for code objects and emit all the ones that we can
-     find (which we hope is very close to 100% of them).
-*/
-#define CODE_ADDR_TO_UID(co)  (((intptr_t)(co)))
-
-#define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0
-
-PyObject* vmprof_eval(PyFrameObject *f, int throwflag);
-
-#ifdef VMPROF_UNIX
-#define VMP_SUPPORTS_NATIVE_PROFILING
-#endif
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-#define MARKER_INTERP_NAME '\x04'   /* deprecated */
-#define MARKER_HEADER '\x05'
-#define MARKER_TIME_N_ZONE '\x06'
-#define MARKER_META '\x07'
-#define MARKER_NATIVE_SYMBOLS '\x08'
-
-#define VERSION_BASE '\x00'
-#define VERSION_THREAD_ID '\x01'
-#define VERSION_TAG '\x02'
-#define VERSION_MEMORY '\x03'
-#define VERSION_MODE_AWARE '\x04'
-#define VERSION_DURATION '\x05'
-#define VERSION_TIMESTAMP '\x06'
-
-#define PROFILE_MEMORY '\x01'
-#define PROFILE_LINES  '\x02'
-#define PROFILE_NATIVE '\x04'
-
-int vmp_write_all(const char *buf, size_t bufsize);
diff --git a/rpython/rlib/rvmprof/src/compat.c b/rpython/rlib/rvmprof/src/compat.c
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/compat.c
+++ /dev/null
@@ -1,137 +0,0 @@
-#include "compat.h"
-
-#include "_vmprof.h"
-
-#if VMPROF_WINDOWS
-#define WIN32_LEAN_AND_MEAN
-#include <Windows.h>
-#endif
-
-static int _vmp_profile_fileno = -1;
-
-int vmp_profile_fileno(void) {
-    return _vmp_profile_fileno;
-}
-void vmp_set_profile_fileno(int fileno) {
-    _vmp_profile_fileno = fileno;
-}
-
-#ifndef VMPROF_WINDOWS
-int vmp_write_all(const char *buf, size_t bufsize)
-{
-    ssize_t count;
-    if (_vmp_profile_fileno == -1) {
-        return -1;
-    }
-    while (bufsize > 0) {
-        count = write(_vmp_profile_fileno, buf, bufsize);
-        if (count <= 0)
-            return -1;   /* failed */
-        buf += count;
-        bufsize -= count;
-    }
-    return 0;
-}
-#endif
-
-int vmp_write_meta(const char * key, const char * value)
-{
-    char marker = MARKER_META;
-    long x = (long)strlen(key);
-    vmp_write_all(&marker, 1);
-    vmp_write_all((char*)&x, sizeof(long));
-    vmp_write_all(key, x);
-    x = (long)strlen(value);
-    vmp_write_all((char*)&x, sizeof(long));
-    vmp_write_all(value, x);
-    return 0;
-}
-
-/**
- * Write the time and zone now.
- */
-
-struct timezone_buf {
-    int64_t tv_sec;
-    int64_t tv_usec;
-};
-#define __SIZE (1+sizeof(struct timezone_buf)+8)
-
-#ifdef VMPROF_UNIX
-int vmp_write_time_now(int marker) {
-    char buffer[__SIZE];
-    struct timezone_buf buf;
-
-    (void)memset(&buffer, 0, __SIZE);
-
-    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
-           "marker must be either a trailer or time_n_zone!");
-
-    struct timeval tv;
-    time_t now;
-    struct tm tm;
-
-
-    /* copy over to the struct */
-    if (gettimeofday(&tv, NULL) != 0) {
-        return -1;
-    }
-    if (time(&now) == (time_t)-1) {
-        return -1;
-    }
-    if (localtime_r(&now, &tm) == NULL) {
-        return -1;
-    }
-    buf.tv_sec = tv.tv_sec;
-    buf.tv_usec = tv.tv_usec;
-    strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
-
-    buffer[0] = marker;
-    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
-    vmp_write_all(buffer, __SIZE);
-    return 0;
-}
-#endif
-
-#ifdef VMPROF_WINDOWS
-int vmp_write_time_now(int marker) {
-    char buffer[__SIZE];
-    struct timezone_buf buf;
-
-    /**
-     * http://stackoverflow.com/questions/10905892/equivalent-of-gettimeday-for-windows
-     */
-
-    // Note: some broken versions only have 8 trailing zero's, the correct
-    // epoch has 9 trailing zero's
-    static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL);
-
-    SYSTEMTIME  system_time;
-    FILETIME    file_time;
-    uint64_t    time;
-
-    (void)memset(&buffer, 0, __SIZE);
-
-    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
-           "marker must be either a trailer or time_n_zone!");
-
-
-    GetSystemTime( &system_time );
-    SystemTimeToFileTime( &system_time, &file_time );
-    time =  ((uint64_t)file_time.dwLowDateTime )      ;
-    time += ((uint64_t)file_time.dwHighDateTime) << 32;
-
-    buf.tv_sec = ((time - EPOCH) / 10000000L);
-    buf.tv_usec = (system_time.wMilliseconds * 1000);
-
-    // time zone not implemented on windows
-    memset(((char*)buffer)+__SIZE-8, 0, 8);
-    (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3);
-
-    buffer[0] = marker;
-    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
-    vmp_write_all(buffer, __SIZE);
-    return 0;
-}
-#endif
-#undef __SIZE
diff --git a/rpython/rlib/rvmprof/src/compat.h b/rpython/rlib/rvmprof/src/compat.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/compat.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#pragma once
-
-#include <Python.h>
-
-#if PY_MAJOR_VERSION >= 3
-    #define PyStr_AS_STRING PyBytes_AS_STRING
-    #define PyStr_GET_SIZE PyBytes_GET_SIZE
-    #define PyStr_NEW      PyUnicode_FromString
-    #define PyLong_NEW     PyLong_FromLong
-#else
-    #define PyStr_AS_STRING PyString_AS_STRING
-    #define PyStr_GET_SIZE PyString_GET_SIZE
-    #define PyStr_NEW      PyString_FromString
-    #define PyLong_NEW     PyInt_FromLong
-#endif
-
-int vmp_write_all(const char *buf, size_t bufsize);
-int vmp_write_time_now(int marker);
-int vmp_write_meta(const char * key, const char * value);
-
-int vmp_profile_fileno(void);
-void vmp_set_profile_fileno(int fileno);
diff --git a/rpython/rlib/rvmprof/src/machine.c b/rpython/rlib/rvmprof/src/machine.c
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/machine.c
+++ /dev/null
@@ -1,40 +0,0 @@
-#include "machine.h"
-
-#include "_vmprof.h"
-#include <stdio.h>
-
-int vmp_machine_bits(void)
-{
-    return sizeof(void*)*8;
-}
-
-const char * vmp_machine_os_name(void)
-{
-#ifdef _WIN32
-   #ifdef _WIN64
-      return "win64";
-   #endif
-  return "win32";
-#elif __APPLE__
-    #include "TargetConditionals.h"
-    #if TARGET_OS_MAC
-        return "mac os x";
-    #endif
-#elif __linux__
-    return "linux";
-#else
-    #error "Unknown compiler"
-#endif
-}
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-#include "libudis86/udis86.h"
-unsigned int vmp_machine_code_instr_length(char* pc)
-{
-    struct ud u;
-    ud_init(&u);
-    ud_set_input_buffer(&u, (uint8_t*)pc, 12);
-    ud_set_mode(&u, vmp_machine_bits());
-    return ud_decode(&u);
-}
-#endif
diff --git a/rpython/rlib/rvmprof/src/machine.h b/rpython/rlib/rvmprof/src/machine.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/machine.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-/**
- * What is the usual word size of the processor? 64bit? 32bit?
- */
-int vmp_machine_bits(void);
-
-/**
- * Return the human readable name of the operating system.
- */
-const char * vmp_machine_os_name(void);
-
-/**
- * How many bytes does the x86 instruction take at pc[0..].
- *
- * Returns 0 on failure.
- */
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-unsigned int vmp_machine_code_instr_length(char* pc);
-#endif
diff --git a/rpython/rlib/rvmprof/src/msiinttypes/changelog.txt b/rpython/rlib/rvmprof/src/msiinttypes/changelog.txt
deleted file mode 100755
--- a/rpython/rlib/rvmprof/src/msiinttypes/changelog.txt
+++ /dev/null
@@ -1,138 +0,0 @@
-------------------------------------------------------------------------
-r26 | 2009-10-02 13:36:47 +0400 | 2 lines
-
-[Issue 5] Change <stdint.h> to "stdint.h" to let compiler search for it in local directory.
-
-------------------------------------------------------------------------
-r25 | 2009-09-17 23:46:49 +0400 | 2 lines
-
-[Issue 4] Fix incorrect int8_t behaviour if compiled with /J flag.
-
-------------------------------------------------------------------------
-r24 | 2009-05-13 14:53:48 +0400 | 2 lines
-
-Forgot about #ifdef __cplusplus guard around 'extern "C"', so inclusion to C files has been broken.
-
-------------------------------------------------------------------------
-r23 | 2009-05-12 01:27:45 +0400 | 3 lines
-
-[Issue 2] Always wrap <wchar&#65533;&#65533; with external "C" {}.
-It turns out that not only Visual Studio 6 requires this, but also newer versions when compiling for ARM.
-
-------------------------------------------------------------------------
-r22 | 2009-05-11 22:22:15 +0400 | 3 lines
-
-[Issue 3] Visual Studio 6 and Embedded Visual C++ 4 doesn't realize that, e.g. char has the same size as __int8 so we give up on __intX for them.
-his should close Issue 3 in issue tracker.
-
-------------------------------------------------------------------------
-r21 | 2008-07-17 09:47:22 +0400 | 4 lines
-
-Get rid of these compiler warnings when compiling for 32-bit:
-  warning C4311: 'type cast' : pointer truncation from 'void *' to 'uintptr_t'
-  warning C4312: 'type cast' : conversion from 'uintptr_t' to 'const void *' of greater size
-
-------------------------------------------------------------------------
-r20 | 2007-10-09 16:54:27 +0400 | 2 lines
-
-Better C99 conformance: macros for format specifiers should only be included in C++ implementations if __STDC_FORMAT_MACROS is defined before <inttypes.h> is included.
-
-------------------------------------------------------------------------
-r19 | 2007-07-04 02:14:40 +0400 | 3 lines
-
-Explicitly cast to appropriate type INT8_MIN, INT16_MIN, INT32_MIN and INT64_MIN constants.
-Due to their unusual definition in Visual Studio headers (-_Ix_MAX-1) they are propagated to int and thus do not have expected type, causing VS6 strict compiler to claim about type inconsistency.
-
-------------------------------------------------------------------------
-r18 | 2007-06-26 16:53:23 +0400 | 2 lines
-
-Better handling of (U)INTx_C macros - now they generate constants of exact width.
-
-------------------------------------------------------------------------
-r17 | 2007-03-29 20:16:14 +0400 | 2 lines
-
-Fix typo: Miscrosoft -> Microsoft.
-
-------------------------------------------------------------------------
-r16 | 2007-02-24 17:32:58 +0300 | 4 lines
-
-Remove <BaseTsd.h> include, as it is not present in Visual Studio 2005 Epxress Edition and required only for INT_PTR and UINT_PTR types.
-
-'intptr_t' and 'uintptr_t' types now defined explicitly with #ifdef _WIN64.
-
-------------------------------------------------------------------------
-r15 | 2007-02-11 20:53:05 +0300 | 2 lines
-
-More correct fix for compilation under VS6.
-
-------------------------------------------------------------------------
-r14 | 2007-02-11 20:04:32 +0300 | 2 lines
-
-Bugfix: fix compiling under VS6, when stdint.h enclosed in 'extern "C" {}'.
-
-------------------------------------------------------------------------
-r13 | 2006-12-13 16:53:11 +0300 | 2 lines
-
-Make _inline modifier for imaxdiv default option. Use STATIC_IMAXDIV to make it static.
-
-------------------------------------------------------------------------
-r12 | 2006-12-13 16:42:24 +0300 | 2 lines
-
-Error message changed: VC6 supported from now.
-
-------------------------------------------------------------------------
-r11 | 2006-12-13 16:39:33 +0300 | 2 lines
-
-All (U)INT* types changed to (unsigned) __int*. This should make stdint.h compatible with VC6.
-
-------------------------------------------------------------------------
-r10 | 2006-12-13 16:20:57 +0300 | 3 lines
-
-Added INLINE_IMAXDIV define switch. 
-If INLINE_IMAXDIV is defined imaxdiv() have static modifier. If not - it is _inline.
-
-------------------------------------------------------------------------
-r9 | 2006-12-13 15:53:52 +0300 | 2 lines
-
-Error message for non-MSC compiler changed.
-
-------------------------------------------------------------------------
-r8 | 2006-12-13 12:47:48 +0300 | 2 lines
-
-Added #ifndef for SIZE_MAX (it is defined in limits.h on MSVSC 8).
-
-------------------------------------------------------------------------
-r7 | 2006-12-13 01:08:02 +0300 | 2 lines
-
-License chaged to BSD-derivative.
-
-------------------------------------------------------------------------
-r6 | 2006-12-13 00:53:20 +0300 | 2 lines
-
-Added <wchar.h> include to avoid warnings when it is included after stdint.h.
-
-------------------------------------------------------------------------
-r5 | 2006-12-12 00:58:05 +0300 | 2 lines
-
-BUGFIX: Definitions of INTPTR_MIN, INTPTR_MAX and UINTPTR_MAX for WIN32 and WIN64 was mixed up.
-
-------------------------------------------------------------------------
-r4 | 2006-12-12 00:51:55 +0300 | 2 lines
-
-Rise #error if _MSC_VER is not defined. I.e. compiler other then Microsoft Visual C++ is used.
-
-------------------------------------------------------------------------
-r3 | 2006-12-11 22:54:14 +0300 | 2 lines
-
-Added <limits.h> include to stdint.h.
-
-------------------------------------------------------------------------
-r2 | 2006-12-11 21:39:27 +0300 | 2 lines
-
-Initial check in.
-
-------------------------------------------------------------------------
-r1 | 2006-12-11 21:30:23 +0300 | 1 line
-
-Initial directory structure.
-------------------------------------------------------------------------
diff --git a/rpython/rlib/rvmprof/src/msiinttypes/inttypes.h b/rpython/rlib/rvmprof/src/msiinttypes/inttypes.h
deleted file mode 100755
--- a/rpython/rlib/rvmprof/src/msiinttypes/inttypes.h
+++ /dev/null
@@ -1,305 +0,0 @@
-// ISO C9x  compliant inttypes.h for Microsoft Visual Studio
-// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 
-// 
-//  Copyright (c) 2006 Alexander Chemeris
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// 
-//   1. Redistributions of source code must retain the above copyright notice,
-//      this list of conditions and the following disclaimer.
-// 
-//   2. Redistributions in binary form must reproduce the above copyright
-//      notice, this list of conditions and the following disclaimer in the
-//      documentation and/or other materials provided with the distribution.
-// 
-//   3. The name of the author may be used to endorse or promote products
-//      derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
-// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
-// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef _MSC_VER // [
-#error "Use this header only with Microsoft Visual C++ compilers!"
-#endif // _MSC_VER ]
-
-#ifndef _MSC_INTTYPES_H_ // [
-#define _MSC_INTTYPES_H_
-
-#if _MSC_VER > 1000
-#pragma once
-#endif
-
-#include "stdint.h"
-
-// 7.8 Format conversion of integer types
-
-typedef struct {
-   intmax_t quot;
-   intmax_t rem;
-} imaxdiv_t;
-
-// 7.8.1 Macros for format specifiers
-
-#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [   See footnote 185 at page 198
-
-// The fprintf macros for signed integers are:
-#define PRId8       "d"
-#define PRIi8       "i"
-#define PRIdLEAST8  "d"
-#define PRIiLEAST8  "i"
-#define PRIdFAST8   "d"
-#define PRIiFAST8   "i"
-
-#define PRId16       "hd"
-#define PRIi16       "hi"
-#define PRIdLEAST16  "hd"
-#define PRIiLEAST16  "hi"
-#define PRIdFAST16   "hd"
-#define PRIiFAST16   "hi"
-
-#define PRId32       "I32d"
-#define PRIi32       "I32i"
-#define PRIdLEAST32  "I32d"
-#define PRIiLEAST32  "I32i"
-#define PRIdFAST32   "I32d"
-#define PRIiFAST32   "I32i"
-
-#define PRId64       "I64d"
-#define PRIi64       "I64i"
-#define PRIdLEAST64  "I64d"
-#define PRIiLEAST64  "I64i"
-#define PRIdFAST64   "I64d"
-#define PRIiFAST64   "I64i"
-
-#define PRIdMAX     "I64d"
-#define PRIiMAX     "I64i"
-
-#define PRIdPTR     "Id"
-#define PRIiPTR     "Ii"
-
-// The fprintf macros for unsigned integers are:
-#define PRIo8       "o"
-#define PRIu8       "u"
-#define PRIx8       "x"
-#define PRIX8       "X"
-#define PRIoLEAST8  "o"
-#define PRIuLEAST8  "u"
-#define PRIxLEAST8  "x"
-#define PRIXLEAST8  "X"
-#define PRIoFAST8   "o"
-#define PRIuFAST8   "u"
-#define PRIxFAST8   "x"
-#define PRIXFAST8   "X"
-
-#define PRIo16       "ho"
-#define PRIu16       "hu"
-#define PRIx16       "hx"
-#define PRIX16       "hX"
-#define PRIoLEAST16  "ho"
-#define PRIuLEAST16  "hu"
-#define PRIxLEAST16  "hx"
-#define PRIXLEAST16  "hX"
-#define PRIoFAST16   "ho"
-#define PRIuFAST16   "hu"
-#define PRIxFAST16   "hx"
-#define PRIXFAST16   "hX"
-
-#define PRIo32       "I32o"
-#define PRIu32       "I32u"
-#define PRIx32       "I32x"
-#define PRIX32       "I32X"
-#define PRIoLEAST32  "I32o"
-#define PRIuLEAST32  "I32u"
-#define PRIxLEAST32  "I32x"
-#define PRIXLEAST32  "I32X"
-#define PRIoFAST32   "I32o"
-#define PRIuFAST32   "I32u"
-#define PRIxFAST32   "I32x"
-#define PRIXFAST32   "I32X"
-
-#define PRIo64       "I64o"
-#define PRIu64       "I64u"
-#define PRIx64       "I64x"
-#define PRIX64       "I64X"
-#define PRIoLEAST64  "I64o"
-#define PRIuLEAST64  "I64u"
-#define PRIxLEAST64  "I64x"
-#define PRIXLEAST64  "I64X"
-#define PRIoFAST64   "I64o"
-#define PRIuFAST64   "I64u"
-#define PRIxFAST64   "I64x"
-#define PRIXFAST64   "I64X"
-
-#define PRIoMAX     "I64o"
-#define PRIuMAX     "I64u"
-#define PRIxMAX     "I64x"
-#define PRIXMAX     "I64X"
-
-#define PRIoPTR     "Io"
-#define PRIuPTR     "Iu"
-#define PRIxPTR     "Ix"
-#define PRIXPTR     "IX"
-
-// The fscanf macros for signed integers are:
-#define SCNd8       "d"
-#define SCNi8       "i"
-#define SCNdLEAST8  "d"
-#define SCNiLEAST8  "i"
-#define SCNdFAST8   "d"
-#define SCNiFAST8   "i"
-
-#define SCNd16       "hd"
-#define SCNi16       "hi"
-#define SCNdLEAST16  "hd"
-#define SCNiLEAST16  "hi"
-#define SCNdFAST16   "hd"
-#define SCNiFAST16   "hi"
-
-#define SCNd32       "ld"
-#define SCNi32       "li"
-#define SCNdLEAST32  "ld"
-#define SCNiLEAST32  "li"
-#define SCNdFAST32   "ld"
-#define SCNiFAST32   "li"
-
-#define SCNd64       "I64d"
-#define SCNi64       "I64i"
-#define SCNdLEAST64  "I64d"
-#define SCNiLEAST64  "I64i"
-#define SCNdFAST64   "I64d"
-#define SCNiFAST64   "I64i"
-
-#define SCNdMAX     "I64d"
-#define SCNiMAX     "I64i"
-
-#ifdef _WIN64 // [
-#  define SCNdPTR     "I64d"
-#  define SCNiPTR     "I64i"
-#else  // _WIN64 ][
-#  define SCNdPTR     "ld"
-#  define SCNiPTR     "li"
-#endif  // _WIN64 ]
-
-// The fscanf macros for unsigned integers are:
-#define SCNo8       "o"
-#define SCNu8       "u"
-#define SCNx8       "x"
-#define SCNX8       "X"
-#define SCNoLEAST8  "o"
-#define SCNuLEAST8  "u"
-#define SCNxLEAST8  "x"
-#define SCNXLEAST8  "X"
-#define SCNoFAST8   "o"
-#define SCNuFAST8   "u"
-#define SCNxFAST8   "x"
-#define SCNXFAST8   "X"
-
-#define SCNo16       "ho"
-#define SCNu16       "hu"
-#define SCNx16       "hx"
-#define SCNX16       "hX"
-#define SCNoLEAST16  "ho"
-#define SCNuLEAST16  "hu"
-#define SCNxLEAST16  "hx"
-#define SCNXLEAST16  "hX"
-#define SCNoFAST16   "ho"
-#define SCNuFAST16   "hu"
-#define SCNxFAST16   "hx"
-#define SCNXFAST16   "hX"
-
-#define SCNo32       "lo"
-#define SCNu32       "lu"
-#define SCNx32       "lx"
-#define SCNX32       "lX"
-#define SCNoLEAST32  "lo"
-#define SCNuLEAST32  "lu"
-#define SCNxLEAST32  "lx"
-#define SCNXLEAST32  "lX"
-#define SCNoFAST32   "lo"
-#define SCNuFAST32   "lu"
-#define SCNxFAST32   "lx"
-#define SCNXFAST32   "lX"
-
-#define SCNo64       "I64o"
-#define SCNu64       "I64u"
-#define SCNx64       "I64x"
-#define SCNX64       "I64X"
-#define SCNoLEAST64  "I64o"
-#define SCNuLEAST64  "I64u"
-#define SCNxLEAST64  "I64x"
-#define SCNXLEAST64  "I64X"
-#define SCNoFAST64   "I64o"
-#define SCNuFAST64   "I64u"
-#define SCNxFAST64   "I64x"
-#define SCNXFAST64   "I64X"
-
-#define SCNoMAX     "I64o"
-#define SCNuMAX     "I64u"
-#define SCNxMAX     "I64x"
-#define SCNXMAX     "I64X"
-
-#ifdef _WIN64 // [
-#  define SCNoPTR     "I64o"
-#  define SCNuPTR     "I64u"
-#  define SCNxPTR     "I64x"
-#  define SCNXPTR     "I64X"
-#else  // _WIN64 ][
-#  define SCNoPTR     "lo"
-#  define SCNuPTR     "lu"
-#  define SCNxPTR     "lx"
-#  define SCNXPTR     "lX"
-#endif  // _WIN64 ]
-
-#endif // __STDC_FORMAT_MACROS ]
-
-// 7.8.2 Functions for greatest-width integer types
-
-// 7.8.2.1 The imaxabs function
-#define imaxabs _abs64
-
-// 7.8.2.2 The imaxdiv function
-
-// This is modified version of div() function from Microsoft's div.c found
-// in %MSVC.NET%\crt\src\div.c
-#ifdef STATIC_IMAXDIV // [
-static
-#else // STATIC_IMAXDIV ][
-_inline
-#endif // STATIC_IMAXDIV ]
-imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom)
-{
-   imaxdiv_t result;
-
-   result.quot = numer / denom;
-   result.rem = numer % denom;
-
-   if (numer < 0 && result.rem > 0) {
-      // did division wrong; must fix up
-      ++result.quot;
-      result.rem -= denom;
-   }
-
-   return result;
-}
-
-// 7.8.2.3 The strtoimax and strtoumax functions
-#define strtoimax _strtoi64
-#define strtoumax _strtoui64
-
-// 7.8.2.4 The wcstoimax and wcstoumax functions
-#define wcstoimax _wcstoi64
-#define wcstoumax _wcstoui64
-
-
-#endif // _MSC_INTTYPES_H_ ]
diff --git a/rpython/rlib/rvmprof/src/msiinttypes/stdint.h b/rpython/rlib/rvmprof/src/msiinttypes/stdint.h
deleted file mode 100755
--- a/rpython/rlib/rvmprof/src/msiinttypes/stdint.h
+++ /dev/null
@@ -1,247 +0,0 @@
-// ISO C9x  compliant stdint.h for Microsoft Visual Studio
-// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 
-// 
-//  Copyright (c) 2006-2008 Alexander Chemeris
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// 
-//   1. Redistributions of source code must retain the above copyright notice,
-//      this list of conditions and the following disclaimer.
-// 
-//   2. Redistributions in binary form must reproduce the above copyright
-//      notice, this list of conditions and the following disclaimer in the
-//      documentation and/or other materials provided with the distribution.
-// 
-//   3. The name of the author may be used to endorse or promote products
-//      derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
-// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
-// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef _MSC_VER // [
-#error "Use this header only with Microsoft Visual C++ compilers!"
-#endif // _MSC_VER ]
-
-#ifndef _MSC_STDINT_H_ // [
-#define _MSC_STDINT_H_
-
-#if _MSC_VER > 1000
-#pragma once
-#endif
-
-#include <limits.h>
-
-// For Visual Studio 6 in C++ mode and for many Visual Studio versions when
-// compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}'
-// or compiler give many errors like this:
-//   error C2733: second C linkage of overloaded function 'wmemchr' not allowed
-#ifdef __cplusplus
-extern "C" {
-#endif
-#  include <wchar.h>
-#ifdef __cplusplus
-}
-#endif
-
-// Define _W64 macros to mark types changing their size, like intptr_t.
-#ifndef _W64
-#  if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300
-#     define _W64 __w64
-#  else
-#     define _W64
-#  endif
-#endif
-
-
-// 7.18.1 Integer types
-
-// 7.18.1.1 Exact-width integer types
-
-// Visual Studio 6 and Embedded Visual C++ 4 doesn't
-// realize that, e.g. char has the same size as __int8
-// so we give up on __intX for them.
-#if (_MSC_VER < 1300)
-   typedef signed char       int8_t;
-   typedef signed short      int16_t;
-   typedef signed int        int32_t;
-   typedef unsigned char     uint8_t;
-   typedef unsigned short    uint16_t;
-   typedef unsigned int      uint32_t;
-#else
-   typedef signed __int8     int8_t;
-   typedef signed __int16    int16_t;
-   typedef signed __int32    int32_t;
-   typedef unsigned __int8   uint8_t;
-   typedef unsigned __int16  uint16_t;
-   typedef unsigned __int32  uint32_t;
-#endif
-typedef signed __int64       int64_t;
-typedef unsigned __int64     uint64_t;
-
-
-// 7.18.1.2 Minimum-width integer types
-typedef int8_t    int_least8_t;
-typedef int16_t   int_least16_t;
-typedef int32_t   int_least32_t;
-typedef int64_t   int_least64_t;
-typedef uint8_t   uint_least8_t;
-typedef uint16_t  uint_least16_t;
-typedef uint32_t  uint_least32_t;
-typedef uint64_t  uint_least64_t;
-
-// 7.18.1.3 Fastest minimum-width integer types
-typedef int8_t    int_fast8_t;
-typedef int16_t   int_fast16_t;
-typedef int32_t   int_fast32_t;
-typedef int64_t   int_fast64_t;
-typedef uint8_t   uint_fast8_t;
-typedef uint16_t  uint_fast16_t;
-typedef uint32_t  uint_fast32_t;
-typedef uint64_t  uint_fast64_t;
-
-// 7.18.1.4 Integer types capable of holding object pointers
-#ifdef _WIN64 // [
-   typedef signed __int64    intptr_t;
-   typedef unsigned __int64  uintptr_t;
-#else // _WIN64 ][
-   typedef _W64 signed int   intptr_t;
-   typedef _W64 unsigned int uintptr_t;
-#endif // _WIN64 ]
-
-// 7.18.1.5 Greatest-width integer types
-typedef int64_t   intmax_t;
-typedef uint64_t  uintmax_t;
-
-
-// 7.18.2 Limits of specified-width integer types
-
-#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [   See footnote 220 at page 257 and footnote 221 at page 259
-
-// 7.18.2.1 Limits of exact-width integer types
-#define INT8_MIN     ((int8_t)_I8_MIN)
-#define INT8_MAX     _I8_MAX
-#define INT16_MIN    ((int16_t)_I16_MIN)
-#define INT16_MAX    _I16_MAX
-#define INT32_MIN    ((int32_t)_I32_MIN)
-#define INT32_MAX    _I32_MAX
-#define INT64_MIN    ((int64_t)_I64_MIN)
-#define INT64_MAX    _I64_MAX
-#define UINT8_MAX    _UI8_MAX
-#define UINT16_MAX   _UI16_MAX
-#define UINT32_MAX   _UI32_MAX
-#define UINT64_MAX   _UI64_MAX
-
-// 7.18.2.2 Limits of minimum-width integer types
-#define INT_LEAST8_MIN    INT8_MIN
-#define INT_LEAST8_MAX    INT8_MAX
-#define INT_LEAST16_MIN   INT16_MIN
-#define INT_LEAST16_MAX   INT16_MAX
-#define INT_LEAST32_MIN   INT32_MIN
-#define INT_LEAST32_MAX   INT32_MAX
-#define INT_LEAST64_MIN   INT64_MIN
-#define INT_LEAST64_MAX   INT64_MAX
-#define UINT_LEAST8_MAX   UINT8_MAX
-#define UINT_LEAST16_MAX  UINT16_MAX
-#define UINT_LEAST32_MAX  UINT32_MAX
-#define UINT_LEAST64_MAX  UINT64_MAX
-
-// 7.18.2.3 Limits of fastest minimum-width integer types
-#define INT_FAST8_MIN    INT8_MIN
-#define INT_FAST8_MAX    INT8_MAX
-#define INT_FAST16_MIN   INT16_MIN
-#define INT_FAST16_MAX   INT16_MAX
-#define INT_FAST32_MIN   INT32_MIN
-#define INT_FAST32_MAX   INT32_MAX
-#define INT_FAST64_MIN   INT64_MIN
-#define INT_FAST64_MAX   INT64_MAX
-#define UINT_FAST8_MAX   UINT8_MAX
-#define UINT_FAST16_MAX  UINT16_MAX
-#define UINT_FAST32_MAX  UINT32_MAX
-#define UINT_FAST64_MAX  UINT64_MAX
-
-// 7.18.2.4 Limits of integer types capable of holding object pointers
-#ifdef _WIN64 // [
-#  define INTPTR_MIN   INT64_MIN
-#  define INTPTR_MAX   INT64_MAX
-#  define UINTPTR_MAX  UINT64_MAX
-#else // _WIN64 ][
-#  define INTPTR_MIN   INT32_MIN
-#  define INTPTR_MAX   INT32_MAX
-#  define UINTPTR_MAX  UINT32_MAX
-#endif // _WIN64 ]
-
-// 7.18.2.5 Limits of greatest-width integer types
-#define INTMAX_MIN   INT64_MIN
-#define INTMAX_MAX   INT64_MAX
-#define UINTMAX_MAX  UINT64_MAX
-
-// 7.18.3 Limits of other integer types
-
-#ifdef _WIN64 // [
-#  define PTRDIFF_MIN  _I64_MIN
-#  define PTRDIFF_MAX  _I64_MAX
-#else  // _WIN64 ][
-#  define PTRDIFF_MIN  _I32_MIN
-#  define PTRDIFF_MAX  _I32_MAX
-#endif  // _WIN64 ]
-
-#define SIG_ATOMIC_MIN  INT_MIN
-#define SIG_ATOMIC_MAX  INT_MAX
-
-#ifndef SIZE_MAX // [
-#  ifdef _WIN64 // [
-#     define SIZE_MAX  _UI64_MAX
-#  else // _WIN64 ][
-#     define SIZE_MAX  _UI32_MAX
-#  endif // _WIN64 ]
-#endif // SIZE_MAX ]
-
-// WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h>
-#ifndef WCHAR_MIN // [
-#  define WCHAR_MIN  0
-#endif  // WCHAR_MIN ]
-#ifndef WCHAR_MAX // [
-#  define WCHAR_MAX  _UI16_MAX
-#endif  // WCHAR_MAX ]
-
-#define WINT_MIN  0
-#define WINT_MAX  _UI16_MAX
-
-#endif // __STDC_LIMIT_MACROS ]
-
-
-// 7.18.4 Limits of other integer types
-
-#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [   See footnote 224 at page 260
-
-// 7.18.4.1 Macros for minimum-width integer constants
-
-#define INT8_C(val)  val##i8
-#define INT16_C(val) val##i16
-#define INT32_C(val) val##i32
-#define INT64_C(val) val##i64
-
-#define UINT8_C(val)  val##ui8
-#define UINT16_C(val) val##ui16
-#define UINT32_C(val) val##ui32
-#define UINT64_C(val) val##ui64
-
-// 7.18.4.2 Macros for greatest-width integer constants
-#define INTMAX_C   INT64_C
-#define UINTMAX_C  UINT64_C
-
-#endif // __STDC_CONSTANT_MACROS ]
-
-
-#endif // _MSC_STDINT_H_ ]
diff --git a/rpython/rlib/rvmprof/src/rss_darwin.h b/rpython/rlib/rvmprof/src/rss_darwin.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/rss_darwin.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/* On OS X we can get RSS using the Mach API. */
-#include <mach/mach.h>
-#include <mach/message.h>
-#include <mach/kern_return.h>
-#include <mach/task_info.h>
-
-static mach_port_t mach_task;
-
-static int setup_rss(void)
-{
-    mach_task = mach_task_self();
-    return 0;
-}
-
-static int teardown_rss(void)
-{
-    return 0;
-}
-
-static long get_current_proc_rss(void)
-{
-    mach_msg_type_number_t out_count = MACH_TASK_BASIC_INFO_COUNT;
-    mach_task_basic_info_data_t taskinfo = { .resident_size = 0 };
-
-    kern_return_t error = task_info(mach_task, MACH_TASK_BASIC_INFO, (task_info_t)&taskinfo, &out_count);
-    if (error == KERN_SUCCESS) {
-        return taskinfo.resident_size / 1024;
-    } else {
-        return -1;
-    }
-}
diff --git a/rpython/rlib/rvmprof/src/rss_unix.h b/rpython/rlib/rvmprof/src/rss_unix.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/rss_unix.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/* On normal Unices we can get RSS from '/proc/<pid>/status'. */
-static int proc_file = -1;
-
-static int setup_rss(void)
-{
-    char buf[128];
-
-    sprintf(buf, "/proc/%d/status", getpid());
-    proc_file = open(buf, O_RDONLY);
-    return proc_file;
-}
-
-static int teardown_rss(void) {
-    close(proc_file);
-    proc_file = -1;
-    return 0;
-}
-
-static long get_current_proc_rss(void)
-{
-    char buf[1024];
-    int i = 0;
-
-    if (lseek(proc_file, 0, SEEK_SET) == -1)
-        return -1;
-    if (read(proc_file, buf, 1024) == -1)
-        return -1;
-    while (i < 1020) {
-        if (strncmp(buf + i, "VmRSS:\t", 7) == 0) {
-            i += 7;
-            return atoi(buf + i);
-        }
-        i++;
-    }
-    return -1;
-}
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -0,0 +1,413 @@
+/*[clinic input]
+module _vmprof
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b443489e38f2be7d]*/
+
+#define _GNU_SOURCE 1
+
+#include <Python.h>
+#include <frameobject.h>
+#include <signal.h>
+
+#include "_vmprof.h"
+
+static volatile int is_enabled = 0;
+
+#if VMPROF_UNIX
+#include "trampoline.h"
+#include "machine.h"
+#include "symboltable.h"
+#include "vmprof_main.h"
+#else
+#include "vmprof_main_win32.h"
+#endif
+#include "stack.h"
+
+static destructor Original_code_dealloc = 0;
+PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
+
+#ifdef VMPROF_UNIX
+#ifdef __clang__
+__attribute__((disable_tail_calls))
+#elif defined(__GNUC__)
+__attribute__((optimize("O1")))
+#endif
+PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag)
+{
+#ifdef X86_64
+    register PY_STACK_FRAME_T * callee_saved asm("rbx");
+#elif defined(X86_32)
+    register PY_STACK_FRAME_T * callee_saved asm("edi");
+#else
+#    error "platform not supported"
+#endif
+
+    asm volatile(
+#ifdef X86_64
+        "movq %1, %0\t\n"
+#elif defined(X86_32)
+        "mov %1, %0\t\n"
+#else
+#    error "platform not supported"
+#endif
+        : "=r" (callee_saved)
+        : "r" (f) );
+    return _default_eval_loop(f, throwflag);
+}
+#endif
+
+static int emit_code_object(PyCodeObject *co)
+{
+    char buf[MAX_FUNC_NAME + 1];
+    char *co_name, *co_filename;
+    int co_firstlineno;
+    int sz;
+#if PY_MAJOR_VERSION >= 3
+    co_name = PyUnicode_AsUTF8(co->co_name);
+    if (co_name == NULL)
+        return -1;
+    co_filename = PyUnicode_AsUTF8(co->co_filename);
+    if (co_filename == NULL)
+        return -1;
+#else
+    co_name = PyString_AS_STRING(co->co_name);
+    co_filename = PyString_AS_STRING(co->co_filename);
+#endif
+    co_firstlineno = co->co_firstlineno;
+
+    sz = snprintf(buf, MAX_FUNC_NAME / 2, "py:%s", co_name);
+    if (sz < 0) sz = 0;
+    if (sz > MAX_FUNC_NAME / 2) sz = MAX_FUNC_NAME / 2;
+    snprintf(buf + sz, MAX_FUNC_NAME / 2, ":%d:%s", co_firstlineno,
+             co_filename);
+    return vmprof_register_virtual_function(buf, CODE_ADDR_TO_UID(co), 500000);
+}
+
+static int _look_for_code_object(PyObject *o, void *all_codes)
+{
+    if (PyCode_Check(o) && !PySet_Contains((PyObject *)all_codes, o)) {
+        Py_ssize_t i;
+        PyCodeObject *co = (PyCodeObject *)o;
+        if (emit_code_object(co) < 0)
+            return -1;
+        if (PySet_Add((PyObject *)all_codes, o) < 0)
+            return -1;
+
+        /* as a special case, recursively look for and add code
+           objects found in the co_consts.  The problem is that code
+           objects are not created as GC-aware in CPython, so we need
+           to hack like this to hope to find most of them. 
+        */
+        i = PyTuple_Size(co->co_consts);
+        while (i > 0) {
+            --i;
+            if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
+                                      all_codes) < 0)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+static void emit_all_code_objects(void)
+{
+    PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
+    Py_ssize_t i, size;
+
+    gc_module = PyImport_ImportModuleNoBlock("gc");
+    if (gc_module == NULL)
+        goto error;
+
+    lst = PyObject_CallMethod(gc_module, "get_objects", "");
+    if (lst == NULL || !PyList_Check(lst))
+        goto error;
+
+    all_codes = PySet_New(NULL);
+    if (all_codes == NULL)
+        goto error;
+
+    size = PyList_GET_SIZE(lst);
+    for (i = 0; i < size; i++) {
+        PyObject *o = PyList_GET_ITEM(lst, i);
+        if (o->ob_type->tp_traverse &&
+            o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes)
+                < 0)
+            goto error;
+    }
+
+ error:
+    Py_XDECREF(all_codes);
+    Py_XDECREF(lst);
+    Py_XDECREF(gc_module);
+}
+
+static void cpyprof_code_dealloc(PyObject *co)
+{
+    if (is_enabled) {
+        emit_code_object((PyCodeObject *)co);
+        /* xxx error return values are ignored */
+    }
+    Original_code_dealloc(co);
+}
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+static void init_cpyprof(int native)
+{
+    // skip this if native should not be enabled
+    if (!native) {
+        vmp_native_disable();
+        return;
+    }
+#if CPYTHON_HAS_FRAME_EVALUATION
+    PyThreadState *tstate = PyThreadState_GET();
+    tstate->interp->eval_frame = vmprof_eval;
+    _default_eval_loop = _PyEval_EvalFrameDefault;
+#else
+    if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx,
+                vmprof_eval, (void*)&_default_eval_loop) == 0) {
+    } else {
+        fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n");
+        // TODO dump the first few bytes and tell them to create an issue!
+        exit(-1);
+    }
+#endif
+    vmp_native_enable();
+}
+#endif
+
+void dump_native_symbols(int fileno)
+{
+    PyObject * mod = NULL;
+
+    mod = PyImport_ImportModuleNoBlock("vmprof");
+    if (mod == NULL)
+        goto error;
+
+    PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
+
+error:
+    Py_XDECREF(mod);
+}
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+static void disable_cpyprof(void)
+{
+    vmp_native_disable();
+#if CPYTHON_HAS_FRAME_EVALUATION
+    PyThreadState *tstate = PyThreadState_GET();
+    tstate->interp->eval_frame = _PyEval_EvalFrameDefault;
+#else
+    if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) {
+        fprintf(stderr, "FATAL: could not remove trampoline\n");
+        exit(-1);
+    }
+#endif
+    dump_native_symbols(vmp_profile_fileno());
+}
+#endif
+
+
+static PyObject *enable_vmprof(PyObject* self, PyObject *args)
+{
+    int fd;
+    int memory = 0;
+    int lines = 0;
+    int native = 0;
+    double interval;
+    char *p_error;
+
+    if (!PyArg_ParseTuple(args, "id|iii", &fd, &interval, &memory, &lines, &native)) {
+        return NULL;
+    }
+    assert(fd >= 0 && "file descripter provided to vmprof must not" \
+                      " be less then zero.");
+
+    if (is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is already enabled");
+        return NULL;
+    }
+
+    vmp_profile_lines(lines);
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    init_cpyprof(native);
+#endif
+
+    if (!Original_code_dealloc) {
+        Original_code_dealloc = PyCode_Type.tp_dealloc;
+        PyCode_Type.tp_dealloc = &cpyprof_code_dealloc;
+    }
+
+    p_error = vmprof_init(fd, interval, memory, lines, "cpython", native);
+    if (p_error) {
+        PyErr_SetString(PyExc_ValueError, p_error);
+        return NULL;
+    }
+
+    if (vmprof_enable(memory) < 0) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    is_enabled = 1;
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static PyObject *
+disable_vmprof(PyObject *module, PyObject *noarg)
+{
+    if (!is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+        return NULL;
+    }
+    is_enabled = 0;
+    vmprof_ignore_signals(1);
+    emit_all_code_objects();
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    disable_cpyprof();
+#endif
+
+    if (vmprof_disable() < 0) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+    if (PyErr_Occurred())
+        return NULL;
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static PyObject *
+write_all_code_objects(PyObject *module, PyObject *noargs)
+{
+    if (!is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+        return NULL;
+    }
+    emit_all_code_objects();
+    if (PyErr_Occurred())
+        return NULL;
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
+static PyObject *
+sample_stack_now(PyObject *module, PyObject *args)
+{
+    PyThreadState * tstate = NULL;
+    PyObject * list;
+    int i;
+    int entry_count;
+    void ** m;
+    void * routine_ip;
+
+    // stop any signal to occur
+    vmprof_ignore_signals(1);
+
+    list = PyList_New(0);
+    if (list == NULL) {
+        goto error;
+    }
+
+    tstate = PyGILState_GetThisThreadState();
+    m = (void**)malloc(SINGLE_BUF_SIZE);
+    if (m == NULL) {
+        PyErr_SetString(PyExc_MemoryError, "could not allocate buffer for stack trace");
+        vmprof_ignore_signals(0);
+        return NULL;
+    }
+    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, 0);
+
+    for (i = 0; i < entry_count; i++) {
+        routine_ip = m[i];
+        PyList_Append(list, PyLong_NEW((long)routine_ip));
+    }
+
+    free(m);
+
+    Py_INCREF(list);
+
+    vmprof_ignore_signals(0);
+    return list;
+error:
+    Py_DECREF(list);
+    Py_INCREF(Py_None);
+
+    vmprof_ignore_signals(0);
+    return Py_None;
+}
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+static PyObject *
+resolve_addr(PyObject *module, PyObject *args) {
+    long long addr;
+    PyObject * o_name = NULL;
+    PyObject * o_lineno = NULL;
+    PyObject * o_srcfile = NULL;
+    char name[128];
+    int lineno = 0;
+    char srcfile[256];
+
+    if (!PyArg_ParseTuple(args, "L", &addr)) {
+        return NULL;
+    }
+    name[0] = '\x00';
+    srcfile[0] = '-';
+    srcfile[1] = '\x00';
+    if (vmp_resolve_addr((void*)addr, name, 128, &lineno, srcfile, 256) != 0) {
+        goto error;
+    }
+
+    o_name = PyStr_NEW(name);
+    if (o_name == NULL) goto error;
+    o_lineno = PyLong_NEW(lineno);
+    if (o_lineno == NULL) goto error;
+    o_srcfile = PyStr_NEW(srcfile);
+    if (o_srcfile == NULL) goto error;
+    //
+    return PyTuple_Pack(3, o_name, o_lineno, o_srcfile);
+error:
+    Py_XDECREF(o_name);
+    Py_XDECREF(o_lineno);
+    Py_XDECREF(o_srcfile);
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+#endif
+
+static PyMethodDef VMProfMethods[] = {
+    {"enable",  enable_vmprof, METH_VARARGS, "Enable profiling."},
+    {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."},
+    {"write_all_code_objects", write_all_code_objects, METH_NOARGS,
+     "Write eagerly all the IDs of code objects"},
+    {"sample_stack_now", sample_stack_now, METH_NOARGS, "Sample the stack now"},
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"},
+#endif
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef VmprofModule = {
+    PyModuleDef_HEAD_INIT,
+    "_vmprof",
+    "",  // doc
+    -1,  // size
+    VMProfMethods
+};
+
+PyMODINIT_FUNC PyInit__vmprof(void)
+{
+    return PyModule_Create(&VmprofModule);
+}
+#else
+PyMODINIT_FUNC init_vmprof(void)
+{
+    Py_InitModule("_vmprof", VMProfMethods);
+}
+#endif
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.h b/rpython/rlib/rvmprof/src/shared/_vmprof.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include "vmprof.h"
+
+#ifdef VMPROF_WINDOWS
+#include "msiinttypes/inttypes.h"
+#include "msiinttypes/stdint.h"
+#else
+#include <inttypes.h>
+#include <stdint.h>
+#endif
+
+/**
+ * This whole setup is very strange. There was just one C file called
+ * _vmprof.c which included all *.h files to copy code. Unsure what
+ * the goal was with this design, but I assume it just 'GREW'
+ *
+ * Thus I'm (plan_rich) slowly trying to separate this. *.h files
+ * should not have complex implementations (all of them currently have them)
+ */
+
+
+#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int))
+
+#define ROUTINE_IS_PYTHON(RIP) ((unsigned long long)RIP & 0x1) == 0
+#define ROUTINE_IS_C(RIP) ((unsigned long long)RIP & 0x1) == 1
+
+/* This returns the address of the code object
+   as the identifier.  The mapping from identifiers to string
+   representations of the code object is done elsewhere, namely:
+
+   * If the code object dies while vmprof is enabled,
+     PyCode_Type.tp_dealloc will emit it.  (We don't handle nicely
+     for now the case where several code objects are created and die
+     at the same memory address.)
+
+   * When _vmprof.disable() is called, then we look around the
+     process for code objects and emit all the ones that we can
+     find (which we hope is very close to 100% of them).
+*/
+#define CODE_ADDR_TO_UID(co)  (((intptr_t)(co)))
+
+#define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0
+
+PyObject* vmprof_eval(PyFrameObject *f, int throwflag);
+
+int vmp_write_all(const char *buf, size_t bufsize);
diff --git a/rpython/rlib/rvmprof/src/shared/compat.c b/rpython/rlib/rvmprof/src/shared/compat.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/compat.c
@@ -0,0 +1,137 @@
+#include "compat.h"
+
+#include "_vmprof.h"
+
+#if VMPROF_WINDOWS
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#endif
+
+static int _vmp_profile_fileno = -1;
+
+int vmp_profile_fileno(void) {
+    return _vmp_profile_fileno;
+}
+void vmp_set_profile_fileno(int fileno) {
+    _vmp_profile_fileno = fileno;
+}
+
+#ifndef VMPROF_WINDOWS
+int vmp_write_all(const char *buf, size_t bufsize)
+{
+    ssize_t count;
+    if (_vmp_profile_fileno == -1) {
+        return -1;
+    }
+    while (bufsize > 0) {
+        count = write(_vmp_profile_fileno, buf, bufsize);
+        if (count <= 0)
+            return -1;   /* failed */
+        buf += count;
+        bufsize -= count;
+    }
+    return 0;
+}
+#endif
+
+int vmp_write_meta(const char * key, const char * value)
+{
+    char marker = MARKER_META;
+    long x = (long)strlen(key);
+    vmp_write_all(&marker, 1);
+    vmp_write_all((char*)&x, sizeof(long));
+    vmp_write_all(key, x);
+    x = (long)strlen(value);
+    vmp_write_all((char*)&x, sizeof(long));
+    vmp_write_all(value, x);
+    return 0;
+}
+
+/**
+ * Write the time and zone now.
+ */
+
+struct timezone_buf {
+    int64_t tv_sec;
+    int64_t tv_usec;
+};
+#define __SIZE (1+sizeof(struct timezone_buf)+8)
+
+#ifdef VMPROF_UNIX
+int vmp_write_time_now(int marker) {
+    char buffer[__SIZE];
+    struct timezone_buf buf;
+
+    (void)memset(&buffer, 0, __SIZE);
+
+    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
+           "marker must be either a trailer or time_n_zone!");
+
+    struct timeval tv;
+    time_t now;
+    struct tm tm;
+
+
+    /* copy over to the struct */
+    if (gettimeofday(&tv, NULL) != 0) {
+        return -1;
+    }
+    if (time(&now) == (time_t)-1) {
+        return -1;
+    }
+    if (localtime_r(&now, &tm) == NULL) {
+        return -1;
+    }
+    buf.tv_sec = tv.tv_sec;
+    buf.tv_usec = tv.tv_usec;
+    strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
+
+    buffer[0] = marker;
+    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
+    vmp_write_all(buffer, __SIZE);
+    return 0;
+}
+#endif
+
+#ifdef VMPROF_WINDOWS
+int vmp_write_time_now(int marker) {
+    char buffer[__SIZE];
+    struct timezone_buf buf;
+
+    /**
+     * http://stackoverflow.com/questions/10905892/equivalent-of-gettimeday-for-windows
+     */
+
+    // Note: some broken versions only have 8 trailing zero's, the correct
+    // epoch has 9 trailing zero's
+    static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL);
+
+    SYSTEMTIME  system_time;
+    FILETIME    file_time;
+    uint64_t    time;
+
+    (void)memset(&buffer, 0, __SIZE);
+
+    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
+           "marker must be either a trailer or time_n_zone!");
+
+
+    GetSystemTime( &system_time );
+    SystemTimeToFileTime( &system_time, &file_time );
+    time =  ((uint64_t)file_time.dwLowDateTime )      ;
+    time += ((uint64_t)file_time.dwHighDateTime) << 32;
+
+    buf.tv_sec = ((time - EPOCH) / 10000000L);
+    buf.tv_usec = (system_time.wMilliseconds * 1000);
+
+    // time zone not implemented on windows
+    memset(((char*)buffer)+__SIZE-8, 0, 8);
+    (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3);
+
+    buffer[0] = marker;
+    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
+    vmp_write_all(buffer, __SIZE);
+    return 0;
+}
+#endif
+#undef __SIZE
diff --git a/rpython/rlib/rvmprof/src/shared/compat.h b/rpython/rlib/rvmprof/src/shared/compat.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/compat.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "vmprof.h"
+
+#ifndef RPYTHON_VMPROF
+#  if PY_MAJOR_VERSION >= 3
+      #define PyStr_AS_STRING PyBytes_AS_STRING
+      #define PyStr_GET_SIZE PyBytes_GET_SIZE
+      #define PyStr_NEW      PyUnicode_FromString
+      #define PyLong_NEW     PyLong_FromLong
+#  else
+      #define PyStr_AS_STRING PyString_AS_STRING
+      #define PyStr_GET_SIZE PyString_GET_SIZE
+      #define PyStr_NEW      PyString_FromString
+      #define PyLong_NEW     PyInt_FromLong
+#  endif
+#endif
+
+int vmp_write_all(const char *buf, size_t bufsize);
+int vmp_write_time_now(int marker);
+int vmp_write_meta(const char * key, const char * value);
+
+int vmp_profile_fileno(void);
+void vmp_set_profile_fileno(int fileno);
diff --git a/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog b/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog
@@ -0,0 +1,602 @@
+2017-01-01  Jakub Jelinek  <jakub at redhat.com>
+
+	Update copyright years.
+
+2016-11-15  Matthias Klose  <doko at ubuntu.com>
+
+	* configure: Regenerate.
+
+2016-09-11  Carlos Liam  <carlos at aarzee.me>
+
+	* all: Remove meaningless trailing whitespace.
+
+2016-05-18  Uros Bizjak  <ubizjak at gmail.com>
+
+	PR target/71161
+	* elf.c (phdr_callback) [__i386__]: Add
+	__attribute__((__force_align_arg_pointer__)).
+
+2016-03-02  Maxim Ostapenko  <m.ostapenko at partner.samsung.com>
+
+	* elf.c (backtrace_initialize): Properly initialize elf_fileline_fn to
+	avoid possible crash.
+	(elf_add): Don't set *fileline_fn to elf_nodebug value in case of
+	missing debug info anymore.
+
+2016-02-06  John David Anglin  <danglin at gcc.gnu.org>
+
+	* mmap.c (MAP_FAILED): Define if not defined.
+
+2016-01-04  Jakub Jelinek  <jakub at redhat.com>
+
+	Update copyright years.
+
+2015-12-18  Andris Pavenis  <andris.pavenis at iki.fi>
+
+	* configure.ac: Specify that DJGPP do not have mmap
+	even when sys/mman.h exists.
+	* configure: Regenerate
+
+2015-12-09  John David Anglin  <danglin at gcc.gnu.org>
+
+	PR libgfortran/68115
+	* configure.ac: Set libbacktrace_cv_sys_sync to no on hppa*-*-hpux*.
+	* configure: Regenerate.
+	* elf.c (backtrace_initialize): Cast __sync_bool_compare_and_swap call
+	to void.
+
+2015-09-17  Ian Lance Taylor  <iant at google.com>
+
+	* posix.c (backtrace_open): Cast second argument of open() to int.
+
+2015-09-11  Ian Lance Taylor  <iant at google.com>
+
+	* Makefile.am (backtrace.lo): Depend on internal.h.
+	(sort.lo, stest.lo): Add explicit dependencies.
+	* Makefile.in: Rebuild.
+
+2015-09-09  Hans-Peter Nilsson  <hp at axis.com>
+
+	* backtrace.c: #include <sys/types.h>.
+
+2015-09-08  Ian Lance Taylor  <iant at google.com>
+
+	PR other/67457
+	* backtrace.c: #include "internal.h".
+	(struct backtrace_data): Add can_alloc field.
+	(unwind): If can_alloc is false, don't try to get file/line
+	information.
+	(backtrace_full): Set can_alloc field in bdata.
+	* alloc.c (backtrace_alloc): Don't call error_callback if it is
+	NULL.
+	* mmap.c (backtrace_alloc): Likewise.
+	* internal.h: Update comments for backtrace_alloc and
+	backtrace_free.
+
+2015-09-08  Ian Lance Taylor  <iant at google.com>
+
+	PR other/67457
+	* mmap.c (backtrace_alloc): Correct test for mmap failure.
+
+2015-08-31  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
+
+	* configure.ac: For spu-*-* targets, set have_fcntl to no.
+	* configure: Regenerate.
+
+2015-08-27  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
+
+	* configure.ac: Remove [disable-shared] argument to LT_INIT.
+	Remove setting PIC_FLAG when building as target library.
+	* configure: Regenerate.
+
+2015-08-26  Hans-Peter Nilsson  <hp at axis.com>
+
+	* configure.ac: Only compile with -fPIC if the target
+	supports it.
+	* configure: Regenerate.
+
+2015-08-24  Ulrich Weigand  <Ulrich.Weigand at de.ibm.com>
+
+	* configure.ac: Set have_mmap to no on spu-*-* targets.
+	* configure: Regenerate.
+
+2015-08-13  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (read_function_entry): Add vec_inlined parameter.
+	Change all callers.
+
+2015-06-11  Martin Sebor  <msebor at redhat.com>
+
+	PR sanitizer/65479
+	* dwarf.c (struct line): Add new field idx.
+	(line_compare): Use it.
+	(add_line): Set it.
+	(read_line_info): Reset it.
+
+2015-05-29  Tristan Gingold  <gingold at adacore.com>
+
+	* pecoff.c: New file.
+	* Makefile.am (FORMAT_FILES): Add pecoff.c and dependencies.
+	* Makefile.in: Regenerate.
+	* filetype.awk: Detect pecoff.
+	* configure.ac: Define BACKTRACE_SUPPORTS_DATA on elf platforms.
+	Add pecoff.
+	* btest.c (test5): Test enabled only if BACKTRACE_SUPPORTS_DATA is
+	true.
+	* backtrace-supported.h.in (BACKTRACE_SUPPORTS_DATA): Define.
+	* configure: Regenerate.
+	* pecoff.c: New file.
+
+2015-05-13  Michael Haubenwallner  <michael.haubenwallner at ssi-schaefer.com>
+
+	* Makefile.in: Regenerated with automake-1.11.6.
+	* aclocal.m4: Likewise.
+	* configure: Likewise.
+
+2015-01-24  Matthias Klose  <doko at ubuntu.com>
+
+	* configure.ac: Move AM_ENABLE_MULTILIB before AC_PROG_CC.
+	* configure: Regenerate.
+
+2015-01-05  Jakub Jelinek  <jakub at redhat.com>
+
+	Update copyright years.
+
+2014-11-21  H.J. Lu  <hongjiu.lu at intel.com>
+
+	PR bootstrap/63784
+	* configure: Regenerated.
+
+2014-11-11  David Malcolm  <dmalcolm at redhat.com>
+
+	* ChangeLog.jit: New.
+
+2014-11-11  Francois-Xavier Coudert  <fxcoudert at gcc.gnu.org>
+
+	PR target/63610
+	* configure: Regenerate.
+
+2014-10-23  Ian Lance Taylor  <iant at google.com>
+
+	* internal.h (backtrace_atomic_load_pointer) [no atomic or sync]:
+	Fix to return void *.
+
+2014-05-08  Ian Lance Taylor  <iant at google.com>
+
+	* mmap.c (backtrace_free): If freeing a large aligned block of
+	memory, call munmap rather than holding onto it.
+	(backtrace_vector_grow): When growing a vector, double the number
+	of pages requested.  When releasing the old version of a grown
+	vector, pass the correct size to backtrace_free.
+
+2014-03-07  Ian Lance Taylor  <iant at google.com>
+
+	* sort.c (backtrace_qsort): Use middle element as pivot.
+
+2014-03-06  Ian Lance Taylor  <iant at google.com>
+
+	* sort.c: New file.
+	* stest.c: New file.
+	* internal.h (backtrace_qsort): Declare.
+	* dwarf.c (read_abbrevs): Call backtrace_qsort instead of qsort.
+	(read_line_info, read_function_entry): Likewise.
+	(read_function_info, build_dwarf_data): Likewise.
+	* elf.c (elf_initialize_syminfo): Likewise.
+	* Makefile.am (libbacktrace_la_SOURCES): Add sort.c.
+	(stest_SOURCES, stest_LDADD): Define.
+	(check_PROGRAMS): Add stest.
+
+2014-02-07  Misty De Meo  <misty at brew.sh>
+
+	PR target/58710
+	* configure.ac: Use AC_LINK_IFELSE in check for
+	_Unwind_GetIPInfo.
+	* configure: Regenerate.
+
+2014-01-02  Richard Sandiford  <rdsandiford at googlemail.com>
+
+	Update copyright years
+
+2013-12-06  Jakub Jelinek  <jakub at redhat.com>
+
+	* elf.c (ET_DYN): Undefine and define again.
+	(elf_add): Add exe argument, if true and ehdr.e_type is ET_DYN,
+	return early -1 without closing the descriptor.
+	(struct phdr_data): Add exe_descriptor.
+	(phdr_callback): If pd->exe_descriptor is not -1, for very first
+	call if dlpi_name is NULL just call elf_add with the exe_descriptor,
+	otherwise backtrace_close the exe_descriptor if not -1.  Adjust
+	call to elf_add.
+	(backtrace_initialize): Adjust call to elf_add.  If it returns
+	-1, set pd.exe_descriptor to descriptor, otherwise set it to -1.
+
+2013-12-05  Ian Lance Taylor  <iant at google.com>
+
+	* alloc.c (backtrace_vector_finish): Add error_callback and data
+	parameters.  Call backtrace_vector_release.  Return address base.
+	* mmap.c (backtrace_vector_finish): Add error_callback and data
+	parameters.  Return address base.
+	* dwarf.c (read_function_info): Get new address base from
+	backtrace_vector_finish.
+	* internal.h (backtrace_vector_finish): Update declaration.
+
+2013-11-27  Ian Lance Taylor  <iant at google.com>
+
+	* dwarf.c (find_address_ranges): New static function, broken out
+	of build_address_map.
+	(build_address_map): Call it.
+	* btest.c (check): Check for missing filename or function, rather
+	than crashing.
+	(f3): Check that enough frames were returned.
+
+2013-11-19  Jakub Jelinek  <jakub at redhat.com>
+
+	* backtrace.h (backtrace_syminfo_callback): Add symsize argument.
+	* elf.c (elf_syminfo): Pass 0 or sym->size to the callback as
+	last argument.
+	* btest.c (struct symdata): Add size field.
+	(callback_three): Add symsize argument.  Copy it to the data->size
+	field.
+	(f23): Set symdata.size to 0.
+	(test5): Likewise.  If sizeof (int) > 1, lookup address of
+	((uintptr_t) &global) + 1.  Verify symdata.val and symdata.size
+	values.
+
+	* atomic.c: Include sys/types.h.
+
+2013-11-18  Ian Lance Taylor  <iant at google.com>
+
+	* configure.ac: Check for support of __atomic extensions.
+	* internal.h: Declare or #define atomic functions for use in
+	backtrace code.
+	* atomic.c: New file.
+	* dwarf.c (dwarf_lookup_pc): Use atomic functions.

From pypy.commits at gmail.com  Mon Feb  6 13:33:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:33:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5898c168.d3811c0a.be7ac.b229@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89977:06a83da024e0
Date: 2017-02-06 13:11 +0100
http://bitbucket.org/pypy/pypy/changeset/06a83da024e0/

Log:	fix test

diff --git a/lib-python/3/test/test_genericpath.py b/lib-python/3/test/test_genericpath.py
--- a/lib-python/3/test/test_genericpath.py
+++ b/lib-python/3/test/test_genericpath.py
@@ -282,7 +282,9 @@
             with self.subTest(attr=attr):
                 with self.assertRaises(ValueError) as cm:
                     getattr(self.pathmodule, attr)('/tmp\x00abcds')
-                self.assertIn('embedded null', str(cm.exception))
+                s = str(cm.exception)
+                if s != 'argument must be a string without NUL characters':
+                    self.assertIn('embedded null', s)
 
 # Following TestCase is not supposed to be run from test_genericpath.
 # It is inherited by other test modules (macpath, ntpath, posixpath).

From pypy.commits at gmail.com  Mon Feb  6 13:33:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:33:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix _lzma.py by copying the 3.5.2->3.5.3
 diff
Message-ID: <5898c16a.84301c0a.98f85.60fd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89978:c0a9a513dbec
Date: 2017-02-06 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/c0a9a513dbec/

Log:	fix _lzma.py by copying the 3.5.2->3.5.3 diff

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -108,7 +108,7 @@
     Always returns True for CHECK_NONE and CHECK_CRC32."""
     return bool(m.lzma_check_is_supported(check))
 
-def catch_lzma_error(fun, *args):
+def catch_lzma_error(fun, *args, ignore_buf_error=False):
     try:
         lzret = fun(*args)
     except:
@@ -124,6 +124,8 @@
     elif lzret == m.LZMA_OPTIONS_ERROR:
         raise LZMAError("Invalid or unsupported options")
     elif lzret == m.LZMA_BUF_ERROR:
+        if ignore_buf_error:
+            return m.LZMA_OK
         raise LZMAError("Insufficient buffer space")
     elif lzret == m.LZMA_PROG_ERROR:
         raise LZMAError("Internal error")
@@ -589,6 +591,9 @@
         lzs.next_in = buf
         lzs.avail_in = buf_len
 
+        if buf_len == 0:
+            return b""
+
         bufsiz = self._bufsiz
         if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
             bufsiz = max_length
@@ -733,7 +738,7 @@
         lzs = self.lzs
 
         lzs.next_in = input_ = ffi.new('uint8_t[]', to_bytes(data))
-        lzs.avail_in = len(data)
+        lzs.avail_in = input_len = len(data)
         outs = [ffi.new('uint8_t[]', BUFSIZ)]
         lzs.next_out, = outs
         lzs.avail_out = BUFSIZ
@@ -742,7 +747,8 @@
 
         while True:
             next_out_pos = int(ffi.cast('intptr_t', lzs.next_out))
-            ret = catch_lzma_error(m.lzma_code, lzs, action)
+            ret = catch_lzma_error(m.lzma_code, lzs, action,
+                      ignore_buf_error=(input_len==0 and lzs.avail_out > 0))
             data_size = int(ffi.cast('intptr_t', lzs.next_out)) - next_out_pos
             if (action == m.LZMA_RUN and lzs.avail_in == 0) or \
                 (action == m.LZMA_FINISH and ret == m.LZMA_STREAM_END):

From pypy.commits at gmail.com  Mon Feb  6 13:33:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:33:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: silence a warning
Message-ID: <5898c16c.4395df0a.8d6ec.09b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89979:82692a33c2d3
Date: 2017-02-06 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/82692a33c2d3/

Log:	silence a warning

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -198,7 +198,7 @@
     filter = parse_filter_spec(filterspec)
     size = ffi.new("uint32_t*")
     catch_lzma_error(m.lzma_properties_size, size, filter)
-    result = ffi.new('char[]', size[0])
+    result = ffi.new('uint8_t[]', size[0])
     catch_lzma_error(m.lzma_properties_encode, filter, result)
     return ffi.buffer(result)[:]
 

From pypy.commits at gmail.com  Mon Feb  6 13:37:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:37:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: dict.fromkeys(): rename the arguments for
 app-level
Message-ID: <5898c279.4a86df0a.244b5.10c1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89980:463f5af4fbf5
Date: 2017-02-06 19:36 +0100
http://bitbucket.org/pypy/pypy/changeset/463f5af4fbf5/

Log:	dict.fromkeys(): rename the arguments for app-level

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -122,7 +122,9 @@
         return w_obj
 
     @staticmethod
-    def descr_fromkeys(space, w_type, w_keys, w_fill=None):
+    def descr_fromkeys(space, w_type, w_iterable, w_value=None):
+        w_keys = w_iterable  # \
+        w_fill = w_value     # / rename the arguments for app-level
         if w_fill is None:
             w_fill = space.w_None
         if space.is_w(w_type, space.w_dict):

From pypy.commits at gmail.com  Mon Feb  6 13:40:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:40:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix OrderedDict.popitem()
Message-ID: <5898c318.0e821c0a.4ff0c.b650@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89981:92da56b2d18e
Date: 2017-02-06 19:39 +0100
http://bitbucket.org/pypy/pypy/changeset/92da56b2d18e/

Log:	fix OrderedDict.popitem()

diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
--- a/lib_pypy/_pypy_collections.py
+++ b/lib_pypy/_pypy_collections.py
@@ -28,7 +28,7 @@
         else:
             it = dict.__iter__(self)
             try:
-                k = it.next()
+                k = next(it)
             except StopIteration:
                 raise KeyError('dictionary is empty')
             return (k, self.pop(k))

From pypy.commits at gmail.com  Mon Feb  6 13:50:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:50:48 -0800 (PST)
Subject: [pypy-commit] pypy default: move_to_end() didn't correctly catch
 KeyError
Message-ID: <5898c588.54b61c0a.33b77.95b2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89982:ed03e78e3f85
Date: 2017-02-06 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/ed03e78e3f85/

Log:	move_to_end() didn't correctly catch KeyError

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1053,7 +1053,10 @@
         if self.is_correct_type(w_key):
             d = self.unerase(w_dict.dstorage)
             key = self.unwrap(w_key)
-            objectmodel.move_to_end(d, key, last_flag)
+            try:
+                objectmodel.move_to_end(d, key, last_flag)
+            except KeyError:
+                w_dict.space.raise_key_error(w_key)
         else:
             self.switch_to_object_strategy(w_dict)
             w_dict.nondescr_move_to_end(w_dict.space, w_key, last_flag)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -307,6 +307,7 @@
                     assert list(d) == other_keys + [key]
                 else:
                     assert list(d) == [key] + other_keys
+                raises(KeyError, __pypy__.move_to_end, d, key * 3, last=last)
 
     def test_keys(self):
         d = {1: 2, 3: 4}
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -783,6 +783,9 @@
     def setdefault(self, key, default):
         return self._dict.setdefault(_r_dictkey(self, key), default)
 
+    def pop(self, key, *default):
+        return self._dict.pop(_r_dictkey(self, key), *default)
+
     def popitem(self):
         dk, value = self._dict.popitem()
         return dk.key, value

From pypy.commits at gmail.com  Mon Feb  6 13:50:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:50:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5898c58a.54b61c0a.33b77.95b9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89983:1a6695039c97
Date: 2017-02-06 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/1a6695039c97/

Log:	hg merge default

diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -14,7 +14,7 @@
 
 * Write them in C++ and bind them through  :doc:`cppyy <cppyy>` using Cling.
 
-* Write them in as `RPython mixed modules`_.
+* Write them as `RPython mixed modules`_.
 
 
 CFFI
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1013,7 +1013,10 @@
         if self.is_correct_type(w_key):
             d = self.unerase(w_dict.dstorage)
             key = self.unwrap(w_key)
-            objectmodel.move_to_end(d, key, last_flag)
+            try:
+                objectmodel.move_to_end(d, key, last_flag)
+            except KeyError:
+                w_dict.space.raise_key_error(w_key)
         else:
             self.switch_to_object_strategy(w_dict)
             w_dict.nondescr_move_to_end(w_dict.space, w_key, last_flag)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -292,6 +292,7 @@
                     assert list(d) == other_keys + [key]
                 else:
                     assert list(d) == [key] + other_keys
+                raises(KeyError, __pypy__.move_to_end, d, key * 3, last=last)
 
     def test_delitem_if_value_is(self):
         import __pypy__
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -783,6 +783,9 @@
     def setdefault(self, key, default):
         return self._dict.setdefault(_r_dictkey(self, key), default)
 
+    def pop(self, key, *default):
+        return self._dict.pop(_r_dictkey(self, key), *default)
+
     def popitem(self):
         dk, value = self._dict.popitem()
         return dk.key, value

From pypy.commits at gmail.com  Mon Feb  6 13:56:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 10:56:45 -0800 (PST)
Subject: [pypy-commit] pypy default: Simplify the signature of pop(). Needed
 anyway because CPython 3.5.3
Message-ID: <5898c6ed.4b371c0a.5e490.bd2d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89984:3dce7b717294
Date: 2017-02-06 19:56 +0100
http://bitbucket.org/pypy/pypy/changeset/3dce7b717294/

Log:	Simplify the signature of pop(). Needed anyway because CPython 3.5.3
	adds a test that OrderedDict.pop() can be called with keyword
	argument 'default'.

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -348,21 +348,15 @@
         w_value = self.getitem(w_key)
         return w_value if w_value is not None else w_default
 
-    @unwrap_spec(defaults_w='args_w')
-    def descr_pop(self, space, w_key, defaults_w):
+    def descr_pop(self, space, w_key, w_default=None):
         """D.pop(k[,d]) -> v, remove specified key and return the
         corresponding value\nIf key is not found, d is returned if given,
         otherwise KeyError is raised
         """
-        len_defaults = len(defaults_w)
-        if len_defaults > 1:
-            raise oefmt(space.w_TypeError,
-                        "pop expected at most 2 arguments, got %d",
-                        1 + len_defaults)
         w_item = self.getitem(w_key)
         if w_item is None:
-            if len_defaults > 0:
-                return defaults_w[0]
+            if w_default is not None:
+                return w_default
             else:
                 space.raise_key_error(w_key)
         else:

From pypy.commits at gmail.com  Mon Feb  6 14:09:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 11:09:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: OrderedDict needs the custom classes for
 keys()/values()/items(), 
Message-ID: <5898c9f2.ec98df0a.78a7a.21a0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89985:f305eb5d13c2
Date: 2017-02-06 20:08 +0100
http://bitbucket.org/pypy/pypy/changeset/f305eb5d13c2/

Log:	OrderedDict needs the custom classes for keys()/values()/items(),
	for reversed() to work on them

diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
--- a/lib_pypy/_pypy_collections.py
+++ b/lib_pypy/_pypy_collections.py
@@ -68,3 +68,30 @@
         return dict.__eq__(self, other)
 
     __ne__ = object.__ne__
+
+    def keys(self):
+        "D.keys() -> a set-like object providing a view on D's keys"
+        return _OrderedDictKeysView(self)
+
+    def items(self):
+        "D.items() -> a set-like object providing a view on D's items"
+        return _OrderedDictItemsView(self)
+
+    def values(self):
+        "D.values() -> an object providing a view on D's values"
+        return _OrderedDictValuesView(self)
+
+
+class _OrderedDictKeysView(KeysView):
+    def __reversed__(self):
+        yield from reversed_dict(self._mapping)
+
+class _OrderedDictItemsView(ItemsView):
+    def __reversed__(self):
+        for key in reversed_dict(self._mapping):
+            yield (key, self._mapping[key])
+
+class _OrderedDictValuesView(ValuesView):
+    def __reversed__(self):
+        for key in reversed_dict(self._mapping):
+            yield self._mapping[key]

From pypy.commits at gmail.com  Mon Feb  6 14:23:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 11:23:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Generalize some tests that check exactly
 how CPython handles
Message-ID: <5898cd3b.6f98df0a.3018e.2280@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89986:03aac3c2ec63
Date: 2017-02-06 20:22 +0100
http://bitbucket.org/pypy/pypy/changeset/03aac3c2ec63/

Log:	Generalize some tests that check exactly how CPython handles
	OrderedDicts, as a subclass of dict that can get messed up manually

diff --git a/lib-python/3/test/test_ordered_dict.py b/lib-python/3/test/test_ordered_dict.py
--- a/lib-python/3/test/test_ordered_dict.py
+++ b/lib-python/3/test/test_ordered_dict.py
@@ -490,15 +490,24 @@
             key = Key()
             od[key] = i
 
-        # These should not crash.
-        with self.assertRaises(KeyError):
+        # These should not crash harder than by raising KeyError
+        # (they do on CPython, but not on PyPy)
+        try:
             list(od.values())
-        with self.assertRaises(KeyError):
+        except KeyError:
+            pass
+        try:
             list(od.items())
-        with self.assertRaises(KeyError):
+        except KeyError:
+            pass
+        try:
             repr(od)
-        with self.assertRaises(KeyError):
+        except KeyError:
+            pass
+        try:
             od.copy()
+        except KeyError:
+            pass
 
     def test_issue24348(self):
         OrderedDict = self.OrderedDict
@@ -549,8 +558,10 @@
         od['spam'] = 1
         od['ham'] = 2
         dict.__delitem__(od, 'spam')
-        with self.assertRaises(KeyError):
+        try:
             repr(od)
+        except KeyError:      # on CPython, not on PyPy
+            pass
 
     def test_dict_clear(self):
         OrderedDict = self.OrderedDict
@@ -566,8 +577,10 @@
         od['spam'] = 1
         od['ham'] = 2
         dict.pop(od, 'spam')
-        with self.assertRaises(KeyError):
+        try:
             repr(od)
+        except KeyError:      # on CPython, not on PyPy
+            pass
 
     def test_dict_popitem(self):
         OrderedDict = self.OrderedDict
@@ -575,8 +588,10 @@
         od['spam'] = 1
         od['ham'] = 2
         dict.popitem(od)
-        with self.assertRaises(KeyError):
+        try:
             repr(od)
+        except KeyError:      # on CPython, not on PyPy
+            pass
 
     def test_dict_setdefault(self):
         OrderedDict = self.OrderedDict
@@ -656,10 +671,14 @@
 
         od = OrderedDict.fromkeys('abcde')
         self.assertEqual(list(od), list('abcde'))
-        with self.assertRaises(RuntimeError):
+        try:
             for i, k in enumerate(od):
                 od.move_to_end(k)
                 self.assertLess(i, 5)
+        except RuntimeError:
+            pass     # XXX on PyPy the change is not detected, as
+                     # the total length of the dict doesn't change
+        od = OrderedDict.fromkeys('bcdea')
         with self.assertRaises(RuntimeError):
             for k in od:
                 od['f'] = None

From pypy.commits at gmail.com  Mon Feb  6 22:09:37 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 06 Feb 2017 19:09:37 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: hg merge py3.5
Message-ID: <58993a71.977d1c0a.d7949.4975@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r89987:062ca5e63c28
Date: 2017-02-05 20:42 +0000
http://bitbucket.org/pypy/pypy/changeset/062ca5e63c28/

Log:	hg merge py3.5

diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -439,3 +439,8 @@
         return (type(self), (self.default_factory,), None, None,
                 iter(self.items()))
 
+
+try:
+    from _pypy_collections import OrderedDict
+except ImportError:
+    pass
diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_collections.py
@@ -0,0 +1,70 @@
+from __pypy__ import reversed_dict, move_to_end
+from _operator import eq as _eq
+from reprlib import recursive_repr as _recursive_repr
+
+class OrderedDict(dict):
+    '''Dictionary that remembers insertion order.
+
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
+
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
+
+    def __reversed__(self):
+        return reversed_dict(self)
+
+    def popitem(self, last=True):
+        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+        Pairs are returned in LIFO order if last is true or FIFO order if false.
+
+        '''
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
+
+    def move_to_end(self, key, last=True):
+        '''Move an existing element to the end (or beginning if last==False).
+
+        Raises KeyError if the element does not exist.
+        When last=True, acts like a fast version of self[key]=self.pop(key).
+
+        '''
+        return move_to_end(self, key, last)
+
+    @_recursive_repr()
+    def __repr__(self):
+        'od.__repr__() <==> repr(od)'
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self.items()))
+
+    def __reduce__(self):
+        'Return state information for pickling'
+        inst_dict = vars(self).copy()
+        return self.__class__, (), inst_dict or None, None, iter(self.items())
+
+    def copy(self):
+        'od.copy() -> a shallow copy of od'
+        return self.__class__(self)
+
+    def __eq__(self, other):
+        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
+        while comparison to a regular mapping is order-insensitive.
+
+        '''
+        if isinstance(other, OrderedDict):
+            return dict.__eq__(self, other) and all(map(_eq, self, other))
+        return dict.__eq__(self, other)
+
+    __ne__ = object.__ne__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -93,6 +93,7 @@
             # ctypes backend: attach these constants to the instance
             self.NULL = self.cast(self.BVoidP, 0)
             self.CData, self.CType = backend._get_types()
+        self.buffer = backend.buffer
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
@@ -316,18 +317,18 @@
         """
         return self._backend.unpack(cdata, length)
 
-    def buffer(self, cdata, size=-1):
-        """Return a read-write buffer object that references the raw C data
-        pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
-        an array.  Can be passed to functions expecting a buffer, or directly
-        manipulated with:
-
-            buf[:]          get a copy of it in a regular string, or
-            buf[idx]        as a single character
-            buf[:] = ...
-            buf[idx] = ...  change the content
-        """
-        return self._backend.buffer(cdata, size)
+   #def buffer(self, cdata, size=-1):
+   #    """Return a read-write buffer object that references the raw C data
+   #    pointed to by the given 'cdata'.  The 'cdata' must be a pointer or
+   #    an array.  Can be passed to functions expecting a buffer, or directly
+   #    manipulated with:
+   #
+   #        buf[:]          get a copy of it in a regular string, or
+   #        buf[idx]        as a single character
+   #        buf[:] = ...
+   #        buf[idx] = ...  change the content
+   #    """
+   #    note that 'buffer' is a type, set on this instance by __init__
 
     def from_buffer(self, python_buffer):
         """Return a <cdata 'char[]'> that points to the data of the
@@ -593,11 +594,15 @@
             ensure('extra_link_args', '/MANIFEST')
 
     def set_source(self, module_name, source, source_extension='.c', **kwds):
+        import os
         if hasattr(self, '_assigned_source'):
             raise ValueError("set_source() cannot be called several times "
                              "per ffi object")
         if not isinstance(module_name, basestring):
             raise TypeError("'module_name' must be a string")
+        if os.sep in module_name or (os.altsep and os.altsep in module_name):
+            raise ValueError("'module_name' must not contain '/': use a dotted "
+                             "name to make a 'package.module' location")
         self._assigned_source = (str(module_name), source,
                                  source_extension, kwds)
 
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -190,7 +190,7 @@
 from GC issues: if the program "forgets" an unfinished greenlet, it will
 always be collected at the next garbage collection.
 
-.. _documentation of the greenlets: http://packages.python.org/greenlet/
+.. _documentation of the greenlets: https://greenlet.readthedocs.io/
 
 
 Unimplemented features
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -78,6 +78,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
+        'move_to_end'               : 'interp_dict.move_to_end',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -44,3 +44,18 @@
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_reversed_dict(space)
+
+ at unwrap_spec(last=bool)
+def move_to_end(space, w_obj, w_key, last=True):
+    """Move the key in a dictionary object into the first or last position.
+
+    This is a __pypy__ function instead of being simply done by calling
+    dict.move_to_end(), for CPython compatibility: dictionaries are only
+    ordered on PyPy.  You should use the collections.OrderedDict class for
+    cases where ordering is important.  That class implements the
+    move_to_end() method by calling __pypy__.move_to_end().
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_move_to_end(space, w_key, last)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -50,7 +50,7 @@
 
         'string': 'func.string',
         'unpack': 'func.unpack',
-        'buffer': 'cbuffer.buffer',
+        'buffer': 'cbuffer.MiniBuffer',
         'memmove': 'func.memmove',
 
         'get_errno': 'cerrno.get_errno',
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -75,18 +75,8 @@
             self.buffer.setslice(start, value.as_str())
 
 
-MiniBuffer.typedef = TypeDef(
-    "_cffi_backend.buffer",
-    __len__ = interp2app(MiniBuffer.descr_len),
-    __getitem__ = interp2app(MiniBuffer.descr_getitem),
-    __setitem__ = interp2app(MiniBuffer.descr_setitem),
-    __weakref__ = make_weakref_descr(MiniBuffer),
-    )
-MiniBuffer.typedef.acceptable_as_base_class = False
-
-
 @unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
-def buffer(space, w_cdata, size=-1):
+def MiniBuffer___new__(space, w_subtype, w_cdata, size=-1):
     ctype = w_cdata.ctype
     if isinstance(ctype, ctypeptr.W_CTypePointer):
         if size < 0:
@@ -107,3 +97,24 @@
                     "don't know the size pointed to by '%s'", ctype.name)
     ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
     return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
+
+MiniBuffer.typedef = TypeDef(
+    "_cffi_backend.buffer",
+    __new__ = interp2app(MiniBuffer___new__),
+    __len__ = interp2app(MiniBuffer.descr_len),
+    __getitem__ = interp2app(MiniBuffer.descr_getitem),
+    __setitem__ = interp2app(MiniBuffer.descr_setitem),
+    __weakref__ = make_weakref_descr(MiniBuffer),
+    __doc__ = """ffi.buffer(cdata[, byte_size]):
+Return a read-write buffer object that references the raw C data
+pointed to by the given 'cdata'.  The 'cdata' must be a pointer or an
+array.  Can be passed to functions expecting a buffer, or directly
+manipulated with:
+
+    buf[:]          get a copy of it in a regular string, or
+    buf[idx]        as a single character
+    buf[:] = ...
+    buf[idx] = ...  change the content
+""",
+    )
+MiniBuffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -265,22 +265,6 @@
         return self.space.wrap(align)
 
 
-    @unwrap_spec(w_cdata=W_CData, size=int)
-    def descr_buffer(self, w_cdata, size=-1):
-        """\
-Return a read-write buffer object that references the raw C data
-ointed to by the given 'cdata'.  The 'cdata' must be a pointer or an
-array.  Can be passed to functions expecting a buffer, or directly
-manipulated with:
-
-    buf[:]          get a copy of it in a regular string, or
-    buf[idx]        as a single character
-    buf[:] = ...
-    buf[idx] = ...  change the content"""
-        #
-        return cbuffer.buffer(self.space, w_cdata, size)
-
-
     @unwrap_spec(w_name=WrappedDefault(None),
                  w_error=WrappedDefault(None),
                  w_onerror=WrappedDefault(None))
@@ -751,6 +735,9 @@
     return space.appexec([], """():
         return type('error', (Exception,), {'__module__': 'ffi'})""")
 
+def make_buffer(space):
+    return space.gettypefor(cbuffer.MiniBuffer)
+
 _extras = get_dict_rtld_constants()
 if sys.platform == 'win32':
     _extras['getwinerror'] = interp2app(W_FFIObject.descr_getwinerror)
@@ -770,7 +757,7 @@
                                      cls=W_FFIObject),
         addressof   = interp2app(W_FFIObject.descr_addressof),
         alignof     = interp2app(W_FFIObject.descr_alignof),
-        buffer      = interp2app(W_FFIObject.descr_buffer),
+        buffer      = ClassAttr(make_buffer),
         callback    = interp2app(W_FFIObject.descr_callback),
         cast        = interp2app(W_FFIObject.descr_cast),
         def_extern  = interp2app(W_FFIObject.descr_def_extern),
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2288,6 +2288,7 @@
     buf = buffer(c)
     assert repr(buf).startswith('<_cffi_backend.buffer object at 0x')
     assert bytes(buf) == b"hi there\x00"
+    assert type(buf) is buffer
     if sys.version_info < (3,):
         assert str(buf) == "hi there\x00"
         assert unicode(buf) == u+"hi there\x00"
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -256,6 +256,10 @@
         ffi = _cffi1_backend.FFI()
         a = ffi.new("signed char[]", [5, 6, 7])
         assert ffi.buffer(a)[:] == b'\x05\x06\x07'
+        assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+        assert type(ffi.buffer(a)) is ffi.buffer
+        assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+        assert type(ffi.buffer(a)) is ffi.buffer
 
     def test_ffi_from_buffer(self):
         import _cffi_backend as _cffi1_backend
diff --git a/pypy/module/_collections/__init__.py b/pypy/module/_collections/__init__.py
--- a/pypy/module/_collections/__init__.py
+++ b/pypy/module/_collections/__init__.py
@@ -25,3 +25,15 @@
         space = self.space
         space.getattr(self, space.wrap('defaultdict'))  # force importing
         space.delattr(self, space.wrap('__missing__'))
+
+    def startup(self, space):
+        # OrderedDict is normally present, but in some cases the line
+        # "from __pypy__ import reversed_dict, move_to_end" from
+        # _pypy_collections.py raises
+        space.appexec([space.wrap(self)], """(mod):
+            try:
+                from _pypy_collections import OrderedDict
+                mod.OrderedDict = OrderedDict
+            except ImportError:
+                pass
+        """)
diff --git a/pypy/module/_collections/test/test_ordereddict.py b/pypy/module/_collections/test/test_ordereddict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_collections/test/test_ordereddict.py
@@ -0,0 +1,8 @@
+
+class AppTestBasic:
+    spaceconfig = dict(usemodules=['_collections'])
+
+    def test_ordereddict_present(self):
+        from _collections import OrderedDict
+        assert issubclass(OrderedDict, dict)
+        assert hasattr(OrderedDict, 'move_to_end')
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -40,6 +40,7 @@
 import pypy.module.cpyext.pyerrors
 import pypy.module.cpyext.typeobject
 import pypy.module.cpyext.object
+import pypy.module.cpyext.buffer
 import pypy.module.cpyext.bytesobject
 import pypy.module.cpyext.bytearrayobject
 import pypy.module.cpyext.tupleobject
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -126,7 +126,7 @@
 Py_TPFLAGS_HEAPTYPE
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
-PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -571,6 +571,7 @@
     '_Py_BuildValue_SizeT', '_Py_VaBuildValue_SizeT',
 
     'PyErr_Format', 'PyErr_NewException', 'PyErr_NewExceptionWithDoc',
+    'PyErr_WarnFormat',
     'PySys_WriteStdout', 'PySys_WriteStderr',
 
     'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction',
@@ -612,6 +613,9 @@
     'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag',
     'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory',
     '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext',
+
+    'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree',
+    'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free',
 ]
 TYPES = {}
 FORWARD_DECLS = []
@@ -1074,7 +1078,8 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI;
+    struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     global_objects = []
@@ -1318,6 +1323,7 @@
                          source_dir / "bytesobject.c",
                          source_dir / "complexobject.c",
                          source_dir / "import.c",
+                         source_dir / "_warnings.c",
                          ]
 
 def build_eci(code, use_micronumpy=False, translating=False):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/buffer.py
@@ -0,0 +1,72 @@
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.interpreter.error import oefmt
+from pypy.module.cpyext.api import (
+    cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP,
+    generic_cpy_call,
+    PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES, PyBUF_SIMPLE)
+from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
+
+ at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
+def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
+    """Returns a pointer to a read-only memory location usable as
+    character-based input.  The obj argument must support the single-segment
+    character buffer interface.  On success, returns 0, sets buffer to the
+    memory location and size to the buffer length.  Returns -1 and sets a
+    TypeError on error.
+    """
+    pto = obj.c_ob_type
+    pb = pto.c_tp_as_buffer
+    if not (pb and pb.c_bf_getbuffer):
+        raise oefmt(space.w_TypeError,
+                    "expected an object with the buffer interface")
+    with lltype.scoped_alloc(Py_buffer) as view:
+        ret = generic_cpy_call(
+            space, pb.c_bf_getbuffer,
+            obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
+        if rffi.cast(lltype.Signed, ret) == -1:
+            return -1
+
+        bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
+        sizep[0] = view.c_len
+
+        if pb.c_bf_releasebuffer:
+            generic_cpy_call(space, pb.c_bf_releasebuffer,
+                             obj, view)
+        Py_DecRef(space, view.c_obj)
+    return 0
+
+
+ at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
+              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
+def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
+    """
+    Fills in a buffer-info structure correctly for an exporter that can only
+    share a contiguous chunk of memory of "unsigned bytes" of the given
+    length. Returns 0 on success and -1 (with raising an error) on error.
+    """
+    flags = rffi.cast(lltype.Signed, flags)
+    if flags & PyBUF_WRITABLE and readonly:
+        raise oefmt(space.w_ValueError, "Object is not writable")
+    view.c_buf = buf
+    view.c_len = length
+    view.c_obj = obj
+    if obj:
+        Py_IncRef(space, obj)
+    view.c_itemsize = 1
+    rffi.setintfield(view, 'c_readonly', readonly)
+    rffi.setintfield(view, 'c_ndim', 1)
+    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+        view.c_format = rffi.str2charp("B")
+    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_ND) == PyBUF_ND:
+        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+        view.c_shape[0] = view.c_len
+    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+        view.c_strides[0] = view.c_itemsize
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
+    return 0
diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/genobject.py
@@ -0,0 +1,7 @@
+from pypy.interpreter.generator import GeneratorIterator, Coroutine
+from pypy.module.cpyext.api import build_type_checkers
+
+
+PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
+
+_, PyCoro_CheckExact = build_type_checkers("Coro", Coroutine)
diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h
--- a/pypy/module/cpyext/include/pymem.h
+++ b/pypy/module/cpyext/include/pymem.h
@@ -7,17 +7,22 @@
 extern "C" {
 #endif
 
-#define PyMem_MALLOC(n)		malloc((n) ? (n) : 1)
-#define PyMem_REALLOC(p, n)	realloc((p), (n) ? (n) : 1)
-#define PyMem_FREE		free
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+#endif
 
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-#define PyMem_Free  PyMem_FREE
-#define PyMem_Realloc  PyMem_REALLOC
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
-#define PyMem_RawMalloc PyMem_Malloc
-#define PyMem_RawFree PyMem_Free
-#define PyMem_RawRealloc PyMem_Realloc
+#define PyMem_MALLOC(n)         PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n)     PyMem_Realloc(p, n)
+#define PyMem_FREE(p)           PyMem_Free(p)
+
 
 /*
  * Type-oriented memory interface
diff --git a/pypy/module/cpyext/include/warnings.h b/pypy/module/cpyext/include/warnings.h
--- a/pypy/module/cpyext/include/warnings.h
+++ b/pypy/module/cpyext/include/warnings.h
@@ -6,6 +6,9 @@
 
 #define PyErr_WarnPy3k(msg, stacklevel) 0
 
+PyAPI_FUNC(int) PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
+                                 const char *format, ...);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,13 +1,11 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
-    PyVarObject, Py_buffer, size_t, slot_function, cts,
-    PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
+    cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t,
+    PyVarObject, size_t, slot_function, cts,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
-    Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
+    Py_GE, CONST_STRING, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
-    PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
-    get_typedescr, _Py_NewReference)
+    PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef, get_typedescr)
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
@@ -16,10 +14,6 @@
 import pypy.module.__builtin__.operation as operation
 
 
-# from include/object.h
-PyBUF_SIMPLE = 0x0000
-PyBUF_WRITABLE = 0x0001
-
 @cpython_api([size_t], rffi.VOIDP)
 def PyObject_Malloc(space, size):
     # returns non-zero-initialized memory, like CPython
@@ -444,36 +438,6 @@
     is active then NULL is returned but PyErr_Occurred() will return false."""
     return space.call_function(space.builtin.get('dir'), w_o)
 
- at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
-def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
-    """Returns a pointer to a read-only memory location usable as
-    character-based input.  The obj argument must support the single-segment
-    character buffer interface.  On success, returns 0, sets buffer to the
-    memory location and size to the buffer length.  Returns -1 and sets a
-    TypeError on error.
-    """
-    pto = obj.c_ob_type
-
-    pb = pto.c_tp_as_buffer
-    if not (pb and pb.c_bf_getbuffer):
-        raise oefmt(space.w_TypeError,
-                    "expected an object with the buffer interface")
-    with lltype.scoped_alloc(Py_buffer) as view:
-        ret = generic_cpy_call(
-            space, pb.c_bf_getbuffer,
-            obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
-        if rffi.cast(lltype.Signed, ret) == -1:
-            return -1
-
-        bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
-        sizep[0] = view.c_len
-
-        if pb.c_bf_releasebuffer:
-            generic_cpy_call(space, pb.c_bf_releasebuffer,
-                             obj, view)
-        Py_DecRef(space, view.c_obj)
-    return 0
-
 # Also in include/object.h
 Py_PRINT_RAW = 1 # No string quotes etc.
 
@@ -493,41 +457,3 @@
     with rffi.scoped_nonmovingbuffer(data) as buf:
         fwrite(buf, 1, count, fp)
     return 0
-
-
-PyBUF_WRITABLE = 0x0001  # Copied from object.h
-
- at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
-              lltype.Signed, lltype.Signed], rffi.INT, error=-1)
-def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
-    """
-    Fills in a buffer-info structure correctly for an exporter that can only
-    share a contiguous chunk of memory of "unsigned bytes" of the given
-    length. Returns 0 on success and -1 (with raising an error) on error.
-    """
-    flags = rffi.cast(lltype.Signed, flags)
-    if flags & PyBUF_WRITABLE and readonly:
-        raise oefmt(space.w_ValueError, "Object is not writable")
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    if obj:
-        Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 1)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
-        view.c_format = rffi.str2charp("B")
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_ND) == PyBUF_ND:
-        view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-        view.c_shape[0] = view.c_len
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
-    if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
-        view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-        view.c_strides[0] = view.c_itemsize
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
-    return 0
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -1,6 +1,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref
+from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rthread
 from rpython.rlib.objectmodel import we_are_translated
@@ -317,3 +318,16 @@
     be held.  The thread state must have been reset with a previous call to
     PyThreadState_Clear()."""
 
+ at cpython_api([], lltype.Void)
+def PyOS_AfterFork(space):
+    """Function to update some internal state after a process fork; this should be
+    called in the new process if the Python interpreter will continue to be used.
+    If a new executable is loaded into the new process, this function does not need
+    to be called."""
+    if not space.config.translation.thread:
+        return
+    from pypy.module.thread import os_thread
+    try:
+        os_thread.reinit_threads(space)
+    except OperationError as e:
+        e.write_unraisable(space, "PyOS_AfterFork()")
diff --git a/pypy/module/cpyext/src/_warnings.c b/pypy/module/cpyext/src/_warnings.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/_warnings.c
@@ -0,0 +1,25 @@
+#include <Python.h>
+
+int
+PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
+                 const char *format, ...)
+{
+    int ret;
+    PyObject *message;
+    va_list vargs;
+
+#ifdef HAVE_STDARG_PROTOTYPES
+    va_start(vargs, format);
+#else
+    va_start(vargs);
+#endif
+    message = PyUnicode_FromFormatV(format, vargs);
+    if (message != NULL) {
+        ret = PyErr_WarnEx(category, PyUnicode_AsUTF8(message), stack_level);
+        Py_DECREF(message);
+    }
+    else
+        ret = -1;
+    va_end(vargs);
+    return ret;
+}
diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c
--- a/pypy/module/cpyext/src/pymem.c
+++ b/pypy/module/cpyext/src/pymem.c
@@ -1,6 +1,86 @@
 #include <Python.h>
 
-void * PyMem_Malloc(size_t n)
+void *
+PyMem_RawMalloc(size_t size)
 {
-    return malloc((n) ? (n) : 1);
+    /*
+     * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
+     * Most python internals blindly use a signed Py_ssize_t to track
+     * things without checking for overflows or negatives.
+     * As size_t is unsigned, checking for size < 0 is not required.
+     */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return malloc(size);
 }
+
+void *
+PyMem_RawCalloc(size_t nelem, size_t elsize)
+{
+    /* see PyMem_RawMalloc() */
+    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+        return NULL;
+    /* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL
+       for calloc(0, 0), which would be treated as an error. Some platforms
+       would return a pointer with no memory behind it, which would break
+       pymalloc.  To solve these problems, allocate an extra byte. */
+    if (nelem == 0 || elsize == 0) {
+        nelem = 1;
+        elsize = 1;
+    }
+    return calloc(nelem, elsize);
+}
+
+void*
+PyMem_RawRealloc(void *ptr, size_t size)
+{
+    /* see PyMem_RawMalloc() */
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return realloc(ptr, size);
+}
+
+void PyMem_RawFree(void *ptr)
+{
+    free(ptr);
+}
+
+
+/* the PyMem_Xxx functions are the same as PyMem_RawXxx in PyPy, for now */
+void *PyMem_Malloc(size_t size)
+{
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return malloc(size);
+}
+
+void *PyMem_Calloc(size_t nelem, size_t elsize)
+{
+    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+        return NULL;
+    if (nelem == 0 || elsize == 0) {
+        nelem = 1;
+        elsize = 1;
+    }
+    return calloc(nelem, elsize);
+}
+
+void* PyMem_Realloc(void *ptr, size_t size)
+{
+    if (size > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+    if (size == 0)
+        size = 1;
+    return realloc(ptr, size);
+}
+
+void PyMem_Free(void *ptr)
+{
+    free(ptr);
+}
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -405,14 +405,6 @@
     (sys.getfilesystemencoding())."""
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, rffi.CCHARP, ], rffi.INT_real, error=-1)
-def PyErr_WarnFormat(space, category, stack_level, format, ):
-    """Function similar to PyErr_WarnEx(), but use
-    PyUnicode_FromFormat() to format the warning message.  format is
-    an ASCII-encoded string.
-    """
-    raise NotImplementedError
-
 
 @cpython_api([rffi.INT_real], rffi.INT_real, error=-1)
 def PySignal_SetWakeupFd(space, fd):
@@ -668,17 +660,6 @@
     extension modules."""
     raise NotImplementedError
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_Check(space, ob):
-    """Return true if ob is a generator object; ob must not be NULL."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyGen_CheckExact(space, ob):
-    """Return true if ob's type is PyGen_Type is a generator object; ob must not
-    be NULL."""
-    raise NotImplementedError
-
 @cpython_api([PyFrameObject], PyObject)
 def PyGen_New(space, frame):
     """Create and return a new generator object based on the frame object. A
@@ -1487,14 +1468,6 @@
     one of the strings '<stdin>' or '???'."""
     raise NotImplementedError
 
- at cpython_api([], lltype.Void)
-def PyOS_AfterFork(space):
-    """Function to update some internal state after a process fork; this should be
-    called in the new process if the Python interpreter will continue to be used.
-    If a new executable is loaded into the new process, this function does not need
-    to be called."""
-    raise NotImplementedError
-
 @cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
 def PyOS_CheckStack(space):
     """Return true when the interpreter runs out of stack space.  This is a reliable
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -0,0 +1,27 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.genobject import PyGen_Check, PyGen_CheckExact
+from pypy.module.cpyext.genobject import PyCoro_CheckExact
+
+
+class TestGenObject(BaseApiTest):
+    def test_genobject(self, space):
+        w_geniter = space.appexec([], """():
+            def f():
+                yield 42
+            return f()
+        """)
+        assert PyGen_Check(space, w_geniter)
+        assert PyGen_CheckExact(space, w_geniter)
+        assert not PyCoro_CheckExact(space, w_geniter)
+        assert not PyGen_Check(space, space.wrap(2))
+        assert not PyGen_CheckExact(space, space.wrap("b"))
+        assert not PyCoro_CheckExact(space, space.wrap([]))
+
+        w_coroutine = space.appexec([], """():
+            async def f():
+                pass
+            return f()
+        """)
+        assert not PyGen_Check(space, w_coroutine)
+        assert not PyGen_CheckExact(space, w_coroutine)
+        assert PyCoro_CheckExact(space, w_coroutine)
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -428,3 +428,18 @@
             assert orig_exc_info == reset_sys_exc_info
             assert new_exc_info == (new_exc.__class__, new_exc, None)
             assert new_exc_info == new_sys_exc_info
+
+    def test_PyErr_WarnFormat(self):
+        import warnings
+
+        module = self.import_extension('foo', [
+                ("test", "METH_NOARGS",
+                 '''
+                 PyErr_WarnFormat(PyExc_UserWarning, 1, "foo %d bar", 42);
+                 Py_RETURN_NONE;
+                 '''),
+                ])
+        with warnings.catch_warnings(record=True) as l:
+            module.test()
+        assert len(l) == 1
+        assert "foo 42 bar" in str(l[0])
diff --git a/pypy/module/cpyext/test/test_pymem.py b/pypy/module/cpyext/test/test_pymem.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_pymem.py
@@ -0,0 +1,34 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class AppTestPyMem(AppTestCpythonExtensionBase):
+    def test_pymem_alloc(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+                int *a, *b;
+                a = PyMem_RawCalloc(4, 50);
+                if (a[49] != 0) {
+                    PyErr_SetString(PyExc_ValueError, "1");
+                    return NULL;
+                }
+                a[49] = 123456;
+                b = PyMem_RawRealloc(a, 2000);
+                b[499] = 789123;
+                PyMem_RawFree(b);
+
+                a = PyMem_Calloc(4, 50);
+                if (a[49] != 0) {
+                    PyErr_SetString(PyExc_ValueError, "2");
+                    return NULL;
+                }
+                a[49] = 123456;
+                b = PyMem_Realloc(a, 2000);
+                b[499] = 789123;
+                PyMem_Free(b);
+
+                Py_RETURN_NONE;
+             """),
+            ])
+        res = module.test()
+        assert res is None
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -548,14 +548,14 @@
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bytes_getbuffer(space, w_str, view, flags):
     from pypy.module.cpyext.bytesobject import PyBytes_AsString
-    from pypy.module.cpyext.object import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
     c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str))
     return PyBuffer_FillInfo(space, view, w_str, c_buf,
                              space.len_w(w_str), 1, flags)
 
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bf_getbuffer(space, w_obj, view, flags):
-    from pypy.module.cpyext.object import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
     buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
     c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     return PyBuffer_FillInfo(space, view, w_obj, c_buf,
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1131,6 +1131,7 @@
             b = ffi.buffer(a)
         except NotImplementedError as e:
             py.test.skip(str(e))
+        assert type(b) is ffi.buffer
         content = b[:]
         assert len(content) == len(b) == 2
         if sys.byteorder == 'little':
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -234,6 +234,7 @@
     a = ffi.new("signed char[]", [5, 6, 7])
     assert ffi.buffer(a)[:] == b'\x05\x06\x07'
     assert ffi.buffer(cdata=a, size=2)[:] == b'\x05\x06'
+    assert type(ffi.buffer(a)) is ffi.buffer
 
 def test_ffi_from_buffer():
     import array
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -37,6 +37,11 @@
                                       ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
+def test_set_source_no_slashes():
+    ffi = FFI()
+    py.test.raises(ValueError, ffi.set_source, "abc/def", None)
+    py.test.raises(ValueError, ffi.set_source, "abc/def", "C code")
+
 
 def test_type_table_func():
     check_type_table("double sin(double);",
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -232,6 +232,37 @@
             w_keys = self.w_keys()
             return space.call_method(w_keys, '__reversed__')
 
+    def nondescr_move_to_end(self, space, w_key, last_flag):
+        """Not exposed directly to app-level, but via __pypy__.move_to_end().
+        """
+        strategy = self.get_strategy()
+        if strategy.has_move_to_end:
+            strategy.move_to_end(self, w_key, last_flag)
+        else:
+            # fall-back
+            w_value = self.getitem(w_key)
+            if w_value is None:
+                space.raise_key_error(w_key)
+            else:
+                self.delitem(w_key)
+                if last_flag:
+                    self.setitem(w_key, w_value)
+                else:
+                    # *very slow* fall-back
+                    keys_w = []
+                    values_w = []
+                    iteratorimplementation = self.iteritems()
+                    while True:
+                        w_k, w_v = iteratorimplementation.next_item()
+                        if w_k is None:
+                            break
+                        keys_w.append(w_k)
+                        values_w.append(w_v)
+                    self.clear()
+                    self.setitem(w_key, w_value)
+                    for i in range(len(keys_w)):
+                        self.setitem(keys_w[i], values_w[i])
+
     def descr_clear(self, space):
         """D.clear() -> None.  Remove all items from D."""
         self.clear()
@@ -499,7 +530,9 @@
         raise NotImplementedError
 
     has_iterreversed = False
-    # no 'getiterreversed': no default implementation available
+    has_move_to_end = False
+    # no 'getiterreversed' and no 'move_to_end': no default
+    # implementation available
 
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         iteritems = self.iteritems(w_dict)
@@ -783,6 +816,9 @@
         dictimpl.iterreversed = iterreversed
         dictimpl.has_iterreversed = True
 
+    if hasattr(dictimpl, 'move_to_end'):
+        dictimpl.has_move_to_end = True
+
     @jit.look_inside_iff(lambda self, w_dict, w_updatedict:
                          w_dict_unrolling_heuristic(w_dict))
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
@@ -962,6 +998,15 @@
     def getiterreversed(self, w_dict):
         return objectmodel.reversed_dict(self.unerase(w_dict.dstorage))
 
+    def move_to_end(self, w_dict, w_key, last_flag):
+        if self.is_correct_type(w_key):
+            d = self.unerase(w_dict.dstorage)
+            key = self.unwrap(w_key)
+            objectmodel.move_to_end(d, key, last_flag)
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.nondescr_move_to_end(w_dict.space, w_key, last_flag)
+
     def prepare_update(self, w_dict, num_extra):
         objectmodel.prepare_dict_update(self.unerase(w_dict.dstorage),
                                         num_extra)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -270,6 +270,29 @@
         del d[key]
         raises(RuntimeError, next, it)
 
+    def test_move_to_end(self):
+        import __pypy__
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo')
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=True)
+        raises(KeyError, __pypy__.move_to_end, {}, 'foo', last=False)
+        def kwdict(**k):
+            return k
+        for last in [False, True]:
+            for d, key in [({1: 2, 3: 4, 5: 6}, 3),
+                           ({"a": 5, "b": 2, "c": 6}, "b"),
+                           (kwdict(d=7, e=8, f=9), "e")]:
+                other_keys = [k for k in d if k != key]
+                __pypy__.move_to_end(d, key, last=last)
+                if not self.on_pypy:
+                    # when running tests on CPython, the underlying
+                    # dicts are not ordered.  We don't get here if
+                    # we're running tests on PyPy or with -A.
+                    assert set(d.keys()) == set(other_keys + [key])
+                elif last:
+                    assert list(d) == other_keys + [key]
+                else:
+                    assert list(d) == [key] + other_keys
+
     def test_keys(self):
         d = {1: 2, 3: 4}
         kys = list(d.keys())
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -14,7 +14,7 @@
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
     SomeFloat, SomeIterator, SomePBC, SomeNone, SomeTypeOf, s_ImpossibleValue,
     s_Bool, s_None, s_Int, unionof, add_knowntypedata,
-    SomeWeakRef, SomeUnicodeString, SomeByteArray)
+    SomeWeakRef, SomeUnicodeString, SomeByteArray, SomeOrderedDict)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
 from rpython.annotator.binaryop import _dict_can_only_throw_keyerror
@@ -575,6 +575,13 @@
         pair(self, s_key).delitem()
     method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
 
+class __extend__(SomeOrderedDict):
+
+    def method_move_to_end(self, s_key, s_last):
+        assert s_Bool.contains(s_last)
+        pair(self, s_key).delitem()
+    method_move_to_end.can_only_throw = _dict_can_only_throw_keyerror
+
 @op.contains.register(SomeString)
 @op.contains.register(SomeUnicodeString)
 def contains_String(annotator, string, char):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,24 @@
         return
     d.delitem_with_hash(key, h)
 
+def _untranslated_move_to_end(d, key, last):
+    "NOT_RPYTHON"
+    value = d.pop(key)
+    if last:
+        d[key] = value
+    else:
+        items = d.items()
+        d.clear()
+        d[key] = value
+        d.update(items)
+
+ at specialize.call_location()
+def move_to_end(d, key, last=True):
+    if not we_are_translated():
+        _untranslated_move_to_end(d, key, last)
+        return
+    d.move_to_end(key, last)
+
 # ____________________________________________________________
 
 def import_from_mixin(M, special_methods=['__init__', '__del__']):
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -602,7 +602,7 @@
 
 if rposix.HAVE_FSTATAT:
     from rpython.rlib.rposix import AT_FDCWD, AT_SYMLINK_NOFOLLOW
-    c_fstatat = rffi.llexternal('fstatat',
+    c_fstatat = rffi.llexternal('fstatat64' if _LINUX else 'fstatat',
         [rffi.INT, rffi.CCHARP, STAT_STRUCT, rffi.INT], rffi.INT,
         compilation_info=compilation_info,
         save_err=rffi.RFFI_SAVE_ERRNO, macro=True)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config, try_inline)
+    fetch_translated_config, try_inline, move_to_end)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -679,6 +679,16 @@
     assert f(29) == 0
     interpret(f, [27])
 
+def test_rordereddict_move_to_end():
+    d = OrderedDict()
+    d['key1'] = 'val1'
+    d['key2'] = 'val2'
+    d['key3'] = 'val3'
+    move_to_end(d, 'key1')
+    assert d.items() == [('key2', 'val2'), ('key3', 'val3'), ('key1', 'val1')]
+    move_to_end(d, 'key1', last=False)
+    assert d.items() == [('key1', 'val1'), ('key2', 'val2'), ('key3', 'val3')]
+
 def test_import_from_mixin():
     class M:    # old-style
         def f(self):
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,15 @@
         hop.exception_is_here()
         hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
 
+    def rtype_method_move_to_end(self, hop):
+        v_dict, v_key, v_last = hop.inputargs(
+            self, self.key_repr, lltype.Bool)
+        if not self.custom_eq_hash:
+            hop.has_implicit_exception(KeyError)  # record that we know about it
+        hop.exception_is_here()
+        hop.gendirectcall(ll_dict_move_to_end, v_dict, v_key, v_last)
+
+
 class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)):
 
     def rtype_getitem((r_dict, r_key), hop):
@@ -542,16 +551,18 @@
         ll_assert(False, "ll_call_insert_clean_function(): invalid lookup_fun")
         assert False
 
-def ll_call_delete_by_entry_index(d, hash, i):
+def ll_call_delete_by_entry_index(d, hash, i, replace_with):
+    # only called from _ll_dict_del, whose @jit.look_inside_iff
+    # condition should control when we get inside here with the jit
     fun = d.lookup_function_no & FUNC_MASK
     if fun == FUNC_BYTE:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_BYTE)
     elif fun == FUNC_SHORT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_SHORT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_SHORT)
     elif IS_64BIT and fun == FUNC_INT:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_INT)
     elif fun == FUNC_LONG:
-        ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
+        ll_dict_delete_by_entry_index(d, hash, i, replace_with, TYPE_LONG)
     else:
         # can't be still FUNC_MUST_REINDEX here
         ll_assert(False, "ll_call_delete_by_entry_index(): invalid lookup_fun")
@@ -805,13 +816,12 @@
     ll_dict_delitem_with_hash(d, key, d.keyhash(key))
 
 def ll_dict_delitem_with_hash(d, key, hash):
-    index = d.lookup_function(d, key, hash, FLAG_DELETE)
+    index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
-    _ll_dict_del(d, index)
+    _ll_dict_del(d, hash, index)
 
- at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
-def _ll_dict_del(d, index):
+def _ll_dict_del_entry(d, index):
     d.entries.mark_deleted(index)
     d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
@@ -823,6 +833,11 @@
     if ENTRIES.must_clear_value:
         entry.value = lltype.nullptr(ENTRY.value.TO)
 
+ at jit.look_inside_iff(lambda d, h, i: jit.isvirtual(d) and jit.isconstant(i))
+def _ll_dict_del(d, hash, index):
+    ll_call_delete_by_entry_index(d, hash, index, DELETED)
+    _ll_dict_del_entry(d, index)
+
     if d.num_live_items == 0:
         # Dict is now empty.  Reset these fields.
         d.num_ever_used_items = 0
@@ -963,7 +978,6 @@
 
 FLAG_LOOKUP = 0
 FLAG_STORE = 1
-FLAG_DELETE = 2
 
 @specialize.memo()
 def _ll_ptr_to_array_of(T):
@@ -985,8 +999,6 @@
     if index >= VALID_OFFSET:
         checkingkey = entries[index - VALID_OFFSET].key
         if direct_compare and checkingkey == key:
-            if store_flag == FLAG_DELETE:
-                indexes[i] = rffi.cast(T, DELETED)
             return index - VALID_OFFSET   # found the entry
         if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
             # correct hash, maybe the key is e.g. a different pointer to
@@ -1000,8 +1012,6 @@
                     # the compare did major nasty stuff to the dict: start over
                     return ll_dict_lookup(d, key, hash, store_flag, T)
             if found:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET
         deletedslot = -1
     elif index == DELETED:
@@ -1030,8 +1040,6 @@
         elif index >= VALID_OFFSET:
             checkingkey = entries[index - VALID_OFFSET].key
             if direct_compare and checkingkey == key:
-                if store_flag == FLAG_DELETE:
-                    indexes[i] = rffi.cast(T, DELETED)
                 return index - VALID_OFFSET   # found the entry
             if d.keyeq is not None and entries.hash(index - VALID_OFFSET) == hash:
                 # correct hash, maybe the key is e.g. a different pointer to
@@ -1044,8 +1052,6 @@
                         # the compare did major nasty stuff to the dict: start over
                         return ll_dict_lookup(d, key, hash, store_flag, T)
                 if found:
-                    if store_flag == FLAG_DELETE:
-                        indexes[i] = rffi.cast(T, DELETED)
                     return index - VALID_OFFSET
         elif deletedslot == -1:
             deletedslot = intmask(i)
@@ -1066,7 +1072,11 @@
         perturb >>= PERTURB_SHIFT
     indexes[i] = rffi.cast(T, index + VALID_OFFSET)
 
-def ll_dict_delete_by_entry_index(d, hash, locate_index, T):
+# the following function is only called from _ll_dict_del, whose
+# @jit.look_inside_iff condition should control when we get inside
+# here with the jit
+ at jit.unroll_safe
+def ll_dict_delete_by_entry_index(d, hash, locate_index, replace_with, T):
     # Another simplified version of ll_dict_lookup() which locates a
     # hashtable entry with the given 'index' stored in it, and deletes it.
     # This *should* be safe against evil user-level __eq__/__hash__
@@ -1083,7 +1093,7 @@
         i = (i << 2) + i + perturb + 1
         i = i & mask
         perturb >>= PERTURB_SHIFT
-    indexes[i] = rffi.cast(T, DELETED)
+    indexes[i] = rffi.cast(T, replace_with)
 
 # ____________________________________________________________
 #
@@ -1253,18 +1263,8 @@
     if hasattr(DICT, 'fnkeyhash'):
         newdict.fnkeyhash = dict.fnkeyhash
 
-    i = 0
-    while i < newdict.num_ever_used_items:
-        d_entry = newdict.entries[i]
-        entry = dict.entries[i]
-        ENTRY = lltype.typeOf(newdict.entries).TO.OF
-        d_entry.key = entry.key
-        if hasattr(ENTRY, 'f_valid'):
-            d_entry.f_valid = entry.f_valid
-        d_entry.value = entry.value
-        if hasattr(ENTRY, 'f_hash'):
-            d_entry.f_hash = entry.f_hash
-        i += 1
+    rgc.ll_arraycopy(dict.entries, newdict.entries, 0, 0,
+                     newdict.num_ever_used_items)
 
     ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
     return newdict
@@ -1390,8 +1390,6 @@
             break
         dic.num_ever_used_items -= 1
 
-    # we must remove the precise entry in the hashtable that points to 'i'
-    ll_call_delete_by_entry_index(dic, entries.hash(i), i)
     return i
 
 def ll_dict_popitem(ELEM, dic):
@@ -1400,21 +1398,139 @@
     r = lltype.malloc(ELEM.TO)
     r.item0 = recast(ELEM.TO.item0, entry.key)
     r.item1 = recast(ELEM.TO.item1, entry.value)
-    _ll_dict_del(dic, i)
+    _ll_dict_del(dic, dic.entries.hash(i), i)
     return r
 
 def ll_dict_pop(dic, key):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         raise KeyError
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value
 
 def ll_dict_pop_default(dic, key, dfl):
-    index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
+    hash = dic.keyhash(key)
+    index = dic.lookup_function(dic, key, hash, FLAG_LOOKUP)
     if index < 0:
         return dfl
     value = dic.entries[index].value
-    _ll_dict_del(dic, index)
+    _ll_dict_del(dic, hash, index)
     return value
+
+def ll_dict_move_to_end(d, key, last):
+    if last:
+        ll_dict_move_to_last(d, key)
+    else:
+        ll_dict_move_to_first(d, key)
+
+def ll_dict_move_to_last(d, key):
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index < 0:
+        raise KeyError
+
+    if old_index == d.num_ever_used_items - 1:
+        return
+
+    # remove the entry at the old position
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    _ll_dict_del_entry(d, old_index)
+
+    # note a corner case: it is possible that 'replace_with' is just too
+    # large to fit in the type T used so far for the index.  But in that
+    # case, the list 'd.entries' is full, and the following call to
+    # _ll_dict_setitem_lookup_done() will necessarily reindex the dict.
+    # So in that case, this value of 'replace_with' should be ignored.
+    ll_call_delete_by_entry_index(d, hash, old_index,
+            replace_with = VALID_OFFSET + d.num_ever_used_items)
+    _ll_dict_setitem_lookup_done(d, key, value, hash, -1)
+
+def ll_dict_move_to_first(d, key):
+    # In this function, we might do a bit more than the strict minimum
+    # of walks over parts of the array, trying to keep the code at least
+    # semi-reasonable, while the goal is still amortized constant-time
+    # over many calls.
+
+    # Call ll_dict_remove_deleted_items() first if there are too many
+    # deleted items.  Not a perfect solution, because lookup_function()
+    # might do random things with the dict and create many new deleted
+    # items.  Still, should be fine, because nothing crucially depends
+    # on this: the goal is to avoid the dictionary's list growing
+    # forever.
+    if d.num_live_items < len(d.entries) // 2 - 16:
+        ll_dict_remove_deleted_items(d)
+
+    hash = d.keyhash(key)
+    old_index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+    if old_index <= 0:
+        if old_index < 0:
+            raise KeyError
+        else:
+            return
+
+    # the goal of the following is to set 'idst' to the number of
+    # deleted entries at the beginning, ensuring 'idst > 0'
+    must_reindex = False
+    if d.entries.valid(0):
+        # the first entry is valid, so we need to make room before.
+        new_allocated = _overallocate_entries_len(d.num_ever_used_items)
+        idst = ((new_allocated - d.num_ever_used_items) * 3) // 4
+        ll_assert(idst > 0, "overallocate did not do enough")
+        newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
+        rgc.ll_arraycopy(d.entries, newitems, 0, idst, d.num_ever_used_items)
+        d.entries = newitems
+        i = 0
+        while i < idst:
+            d.entries.mark_deleted(i)
+            i += 1
+        d.num_ever_used_items += idst
+        old_index += idst
+        must_reindex = True
+        idst -= 1
+    else:
+        idst = d.lookup_function_no >> FUNC_SHIFT
+        # All entries in range(0, idst) are deleted.  Check if more are
+        while not d.entries.valid(idst):
+            idst += 1
+        if idst == old_index:
+            d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                    (old_index << FUNC_SHIFT))
+            return
+        idst -= 1
+        d.lookup_function_no = ((d.lookup_function_no & FUNC_MASK) |
+                                (idst << FUNC_SHIFT))
+
+    # remove the entry at the old position
+    ll_assert(d.entries.valid(old_index),
+              "ll_dict_move_to_first: lost old_index")
+    ENTRY = lltype.typeOf(d.entries).TO.OF
+    old_entry = d.entries[old_index]
+    key = old_entry.key
+    value = old_entry.value
+    if hasattr(ENTRY, 'f_hash'):
+        ll_assert(old_entry.f_hash == hash,
+                  "ll_dict_move_to_first: bad hash")
+    _ll_dict_del_entry(d, old_index)
+
+    # put the entry at its new position
+    ll_assert(not d.entries.valid(idst),
+              "ll_dict_move_to_first: overwriting idst")
+    new_entry = d.entries[idst]
+    new_entry.key = key
+    new_entry.value = value
+    if hasattr(ENTRY, 'f_hash'):
+        new_entry.f_hash = hash
+    if hasattr(ENTRY, 'f_valid'):
+        new_entry.f_valid = True
+    d.num_live_items += 1
+
+    # fix the index
+    if must_reindex:
+        ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+    else:
+        ll_call_delete_by_entry_index(d, hash, old_index,
+                replace_with = VALID_OFFSET + idst)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1188,6 +1188,12 @@
         assert not self.ll_contains(self.l_dict, ll_key)
         self.removed_keys.append(key)
 
+    def move_to_end(self, key, last=True):
+        "For test_rordereddict"
+
+    def move_to_first(self, key):
+        self.move_to_end(key, last=False)
+
     def copydict(self):
         self.l_dict = self.ll_copy(self.l_dict)
         assert self.ll_len(self.l_dict) == len(self.reference)
@@ -1250,6 +1256,15 @@
         return builds(Action,
             just('delitem'), tuples(sampled_from(self.space.reference)))
 
+    def st_move_to_end(self):
+        return builds(Action,
+            just('move_to_end'), tuples(sampled_from(self.space.reference)))
+
+    def st_move_to_first(self):
+        return builds(Action,
+            just('move_to_first'),
+                tuples(sampled_from(self.space.reference)))
+
     def steps(self):
         if not self.space:
             return builds(Action, just('setup'), tuples(st_keys, st_values))
@@ -1258,7 +1273,8 @@
         if self.space.reference:
             return (
                 self.st_setitem() | sampled_from(global_actions) |
-                self.st_updateitem() | self.st_delitem())
+                self.st_updateitem() | self.st_delitem() |
+                self.st_move_to_end() | self.st_move_to_first())
         else:
             return (self.st_setitem() | sampled_from(global_actions))
 
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -1,7 +1,8 @@
 import py
+import random
 from collections import OrderedDict
 
-from hypothesis import settings
+from hypothesis import settings, given, strategies
 from hypothesis.stateful import run_state_machine_as_test
 
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -145,14 +146,18 @@
         ll_d = rordereddict.ll_newdict(DICT)
         rordereddict.ll_dict_setitem(ll_d, llstr("k"), 1)
         rordereddict.ll_dict_setitem(ll_d, llstr("j"), 2)
-        ITER = rordereddict.get_ll_dictiter(lltype.Ptr(DICT))
+        assert [hlstr(entry.key) for entry in self._ll_iter(ll_d)] == ["k", "j"]
+
+    def _ll_iter(self, ll_d):
+        ITER = rordereddict.get_ll_dictiter(lltype.typeOf(ll_d))
         ll_iter = rordereddict.ll_dictiter(ITER, ll_d)
         ll_dictnext = rordereddict._ll_dictnext
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "k"
-        num = ll_dictnext(ll_iter)
-        assert hlstr(ll_d.entries[num].key) == "j"
-        py.test.raises(StopIteration, ll_dictnext, ll_iter)
+        while True:
+            try:
+                num = ll_dictnext(ll_iter)
+            except StopIteration:
+                break
+            yield ll_d.entries[num]
 
     def test_popitem(self):
         DICT = self._get_str_dict()
@@ -337,6 +342,31 @@
             num_nonfrees += (got > 0)
         assert d.resize_counter <= idx.getlength() * 2 - num_nonfrees * 3
 
+    @given(strategies.lists(strategies.integers(min_value=1, max_value=5)))
+    def test_direct_move_to_end(self, lst):
+        DICT = self._get_int_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        rordereddict.ll_dict_setitem(ll_d, 1, 11)
+        rordereddict.ll_dict_setitem(ll_d, 2, 22)
+        def content():
+            return [(entry.key, entry.value) for entry in self._ll_iter(ll_d)]
+        for case in lst:
+            if case == 1:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, True)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 2:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, True)
+                assert content() == [(1, 11), (2, 22)]
+            elif case == 3:
+                py.test.raises(KeyError, rordereddict.ll_dict_move_to_end,
+                                                 ll_d, 3, True)
+            elif case == 4:
+                rordereddict.ll_dict_move_to_end(ll_d, 2, False)
+                assert content() == [(2, 22), (1, 11)]
+            elif case == 5:
+                rordereddict.ll_dict_move_to_end(ll_d, 1, False)
+                assert content() == [(1, 11), (2, 22)]
+
 
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
@@ -369,10 +399,29 @@
         res = self.interpret(func, [5])
         assert res == 6
 
+    def test_move_to_end(self):
+        def func():
+            d1 = OrderedDict()
+            d1['key1'] = 'value1'
+            d1['key2'] = 'value2'
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key1')
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key2')
+                assert d1.keys() == ['key1', 'key2']
+            for i in range(20):
+                objectmodel.move_to_end(d1, 'key2', last=False)
+                assert d1.keys() == ['key2', 'key1']
+                objectmodel.move_to_end(d1, 'key1', last=False)
+                assert d1.keys() == ['key1', 'key2']
+        func()
+        self.interpret(func, [])
+
 
 class ODictSpace(MappingSpace):
     MappingRepr = rodct.OrderedDictRepr
     new_reference = OrderedDict
+    moved_around = False
     ll_getitem = staticmethod(rodct.ll_dict_getitem)
     ll_setitem = staticmethod(rodct.ll_dict_setitem)
     ll_delitem = staticmethod(rodct.ll_dict_delitem)
@@ -417,9 +466,23 @@
     def removeindex(self):
         # remove the index, as done during translation for prebuilt dicts
         # (but cannot be done if we already removed a key)
-        if not self.removed_keys:
+        if not self.removed_keys and not self.moved_around:
             rodct.ll_no_initial_index(self.l_dict)
 
+    def move_to_end(self, key, last=True):
+        ll_key = self.ll_key(key)
+        rodct.ll_dict_move_to_end(self.l_dict, ll_key, last)
+        value = self.reference.pop(key)
+        if last:
+            self.reference[key] = value
+        else:
+            items = self.reference.items()
+            self.reference.clear()
+            self.reference[key] = value
+            self.reference.update(items)
+        # prevent ll_no_initial_index()
+        self.moved_around = True
+
     def fullcheck(self):
         # overridden to also check key order
         assert self.ll_len(self.l_dict) == len(self.reference)

From pypy.commits at gmail.com  Mon Feb  6 22:09:39 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 06 Feb 2017 19:09:39 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: Test and fix for the fields of
 the Py_buffer not being set correctly in our bf_getbuffer
Message-ID: <58993a73.16a1df0a.d4aea.a65e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r89988:6f7a4aa078bd
Date: 2017-02-07 03:08 +0000
http://bitbucket.org/pypy/pypy/changeset/6f7a4aa078bd/

Log:	Test and fix for the fields of the Py_buffer not being set correctly
	in our bf_getbuffer

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,10 +1,10 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP,
+    cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP, cts,
     generic_cpy_call,
     PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES, PyBUF_SIMPLE)
-from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import PyObject, incref, Py_DecRef
 
 @cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
 def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
@@ -35,6 +35,33 @@
         Py_DecRef(space, view.c_obj)
     return 0
 
+def fill_buffer(space, view, pybuf, py_obj):
+    view.c_buf = cts.cast('void *', pybuf.get_raw_address())
+    view.c_obj = py_obj
+    if py_obj:
+        incref(space, py_obj)
+    view.c_len = pybuf.getlength()
+    view.c_itemsize = pybuf.getitemsize()
+    rffi.setintfield(view, 'c_readonly', int(pybuf.readonly))
+    rffi.setintfield(view, 'c_ndim', pybuf.getndim())
+    view.c_format = rffi.str2charp(pybuf.getformat())
+    shape = pybuf.getshape()
+    if not shape:
+        view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    else:
+        view.c_shape = cts.cast('Py_ssize_t*', view.c__shape)
+        for i, n in enumerate(shape):
+            view.c_shape[i] = n
+    strides = pybuf.getstrides()
+    if not strides:
+        view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    else:
+        view.c_strides = cts.cast('Py_ssize_t*', view.c__strides)
+        for i, n in enumerate(strides):
+            view.c_strides[i] = n
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
 
 @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
               lltype.Signed, lltype.Signed], rffi.INT, error=-1)
@@ -51,7 +78,7 @@
     view.c_len = length
     view.c_obj = obj
     if obj:
-        Py_IncRef(space, obj)
+        incref(space, obj)
     view.c_itemsize = 1
     rffi.setintfield(view, 'c_readonly', readonly)
     rffi.setintfield(view, 'c_ndim', 1)
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2871,7 +2871,17 @@
     return PyBytes_FromStringAndSize((char*)ptr, size);
 }
 
-
+static PyObject *
+write_buffer_len(PyObject * self, PyObject * obj)
+{
+    void* buf;
+    Py_ssize_t buf_len;
+    if (PyObject_AsWriteBuffer(obj, &buf, &buf_len) < 0) {
+        PyErr_SetString(PyExc_ValueError, "bad value");
+        return NULL;
+    }
+    return PyLong_FromLong(buf_len);
+}
 
 /*********************** Install Module **************************/
 
@@ -2880,6 +2890,7 @@
      PyDoc_STR("Internal. Used for pickling support.")},
     {"switch_multiply",   (PyCFunction)switch_multiply, METH_NOARGS, NULL},
     {"readbuffer_as_string",   (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL},
+    {"write_buffer_len", write_buffer_len, METH_O, NULL},
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -84,6 +84,9 @@
         arr = module.array('u', '123')
         view = memoryview(arr)
         assert view.itemsize == 4
+        assert module.write_buffer_len(arr) == 12
+        assert len(module.readbuffer_as_string(arr)) == 12
+        assert len(module.readbuffer_as_string(view)) == 12
 
     def test_subclass(self):
         import struct
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -555,11 +555,10 @@
 
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bf_getbuffer(space, w_obj, view, flags):
-    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import fill_buffer
     buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
-    c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-    return PyBuffer_FillInfo(space, view, w_obj, c_buf,
-                             space.len_w(w_obj), 0, flags)
+    fill_buffer(space, view, buf, as_pyobj(space, w_obj))
+    return 0
 
 def setup_buffer_procs(space, w_type, pto):
     bufspec = w_type.layout.typedef.buffer

From pypy.commits at gmail.com  Tue Feb  7 02:38:28 2017
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 06 Feb 2017 23:38:28 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: unclear travel dates,
 but reserving a bed for the week
Message-ID: <58997974.cd2f1c0a.44bd1.eee6@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: extradoc
Changeset: r5774:6af83e7a99d4
Date: 2017-02-07 08:38 +0100
http://bitbucket.org/pypy/extradoc/changeset/6af83e7a99d4/

Log:	unclear travel dates, but reserving a bed for the week

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -11,6 +11,7 @@
 ==================== ============== =======================
 Armin Rigo                          private
 Richard Plangger     26.02/04.02    Ermina
+Remi Meier           (?)27.02/04.02 Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Tue Feb  7 02:47:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 06 Feb 2017 23:47:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <58997b95.cda0df0a.a56a.ef6f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89989:7f05167cc42c
Date: 2017-02-07 08:46 +0100
http://bitbucket.org/pypy/pypy/changeset/7f05167cc42c/

Log:	fix

diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
--- a/lib_pypy/_pypy_collections.py
+++ b/lib_pypy/_pypy_collections.py
@@ -1,6 +1,8 @@
 from __pypy__ import reversed_dict, move_to_end
 from _operator import eq as _eq
 from reprlib import recursive_repr as _recursive_repr
+import _collections_abc
+
 
 class OrderedDict(dict):
     '''Dictionary that remembers insertion order.
@@ -82,16 +84,16 @@
         return _OrderedDictValuesView(self)
 
 
-class _OrderedDictKeysView(KeysView):
+class _OrderedDictKeysView(_collections_abc.KeysView):
     def __reversed__(self):
         yield from reversed_dict(self._mapping)
 
-class _OrderedDictItemsView(ItemsView):
+class _OrderedDictItemsView(_collections_abc.ItemsView):
     def __reversed__(self):
         for key in reversed_dict(self._mapping):
             yield (key, self._mapping[key])
 
-class _OrderedDictValuesView(ValuesView):
+class _OrderedDictValuesView(_collections_abc.ValuesView):
     def __reversed__(self):
         for key in reversed_dict(self._mapping):
             yield self._mapping[key]

From pypy.commits at gmail.com  Tue Feb  7 03:23:43 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 07 Feb 2017 00:23:43 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: test_ztranslation translates now
Message-ID: <5899840f.068f1c0a.ef4bf.a96e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89990:ad7e7ef86e35
Date: 2017-02-07 09:23 +0100
http://bitbucket.org/pypy/pypy/changeset/ad7e7ef86e35/

Log:	test_ztranslation translates now

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -15,8 +15,8 @@
 ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof')
 SRC = ROOT.join('src')
 SHARED = SRC.join('shared')
-UDIS86 = ROOT.join('libudis86')
-BACKTRACE = ROOT.join('libbacktrace')
+UDIS86 = SHARED.join('libudis86')
+BACKTRACE = SHARED.join('libbacktrace')
 
 compile_extra = ['-DRPYTHON_LL2CTYPES','-DRPYTHON_VMPROF']
 if sys.platform.startswith('linux'):
@@ -31,7 +31,7 @@
        BACKTRACE.join('posix.c'),
        BACKTRACE.join('sort.c'),
     ]
-    _libs = ['dl']
+    _libs = ['dl', 'unwind']
     compile_extra += ['-DVMPROF_UNIX']
     compile_extra += ['-DVMPROF_LINUX']
 elif sys.platform == 'darwin':
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -122,7 +122,7 @@
         self._gather_all_code_objs = gather_all_code_objs
 
     @jit.dont_look_inside
-    def enable(self, fileno, interval):
+    def enable(self, fileno, interval, memory=0):
         """Enable vmprof.  Writes go to the given 'fileno'.
         The sampling interval is given by 'interval' as a number of
         seconds, as a float which must be smaller than 1.0.
@@ -132,12 +132,15 @@
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
 
-        p_error = self.cintf.vmprof_init(fileno, interval, "pypy")
+        lines = 0
+        native = 0
+
+        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native)
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable()
+        res = self.cintf.vmprof_enable(memory)
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -17,3 +17,33 @@
 #else
 #include "shared/vmprof_main_win32.h"
 #endif
+
+#ifdef VMPROF_UNIX
+#ifdef __clang__
+__attribute__((disable_tail_calls))
+#elif defined(__GNUC__)
+__attribute__((optimize("O1")))
+#endif
+PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag)
+{
+#ifdef X86_64
+    register PY_STACK_FRAME_T * callee_saved asm("rbx");
+#elif defined(X86_32)
+    register PY_STACK_FRAME_T * callee_saved asm("edi");
+#else
+#    error "platform not supported"
+#endif
+
+    asm volatile(
+#ifdef X86_64
+        "movq %1, %0\t\n"
+#elif defined(X86_32)
+        "mov %1, %0\t\n"
+#else
+#    error "platform not supported"
+#endif
+        : "=r" (callee_saved)
+        : "r" (f) );
+    return NULL; // TODO _default_eval_loop(f, throwflag);
+}
+#endif
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.h b/rpython/rlib/rvmprof/src/shared/_vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.h
@@ -42,6 +42,4 @@
 
 #define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0
 
-PyObject* vmprof_eval(PyFrameObject *f, int throwflag);
-
 int vmp_write_all(const char *buf, size_t bufsize);
diff --git a/rpython/rlib/rvmprof/src/shared/stack.c b/rpython/rlib/rvmprof/src/shared/stack.c
--- a/rpython/rlib/rvmprof/src/shared/stack.c
+++ b/rpython/rlib/rvmprof/src/shared/stack.c
@@ -6,8 +6,10 @@
 
 #include <string.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/types.h>
 #include <stddef.h>
+#include <assert.h>
 
 #include "vmprof.h"
 #include "compat.h"
@@ -59,7 +61,6 @@
     int j;
     long line;
     char *lnotab;
-    intptr_t addr;
 
 #ifndef RPYTHON_VMPROF // pypy does not support line profiling
     if (vmp_profiles_python_lines()) {
@@ -99,26 +100,20 @@
 #else
     //result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
     //*depth = *depth + 1;
+
+    if (frame->kind == VMPROF_CODE_TAG) {
+        int n = *depth;
+        result[n++] = (void*)frame->kind;
+        result[n++] = (void*)frame->value;
+        *depth = n;
+    }
 #ifdef PYPY_JIT_CODEMAP
-    if (pypy_find_codemap_at_addr((intptr_t)pc, &addr)) {
-        // the bottom part is jitted, means we can fill up the first part
-        // from the JIT
-        *depth = vmprof_write_header_for_jit_addr(result, *depth, pc, max_depth);
-        stack = stack->next; // skip the first item as it contains garbage
+    else if (frame->kind == VMPROF_JITTED_TAG) {
+        intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
+        n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
     }
 #endif
-    while (n < max_depth - 1 && stack) {
-        if (stack->kind == VMPROF_CODE_TAG) {
-            result[n] = stack->kind;
-            result[n + 1] = stack->value;
-            n += 2;
-        }
-#ifdef PYPY_JIT_CODEMAP
-        else if (stack->kind == VMPROF_JITTED_TAG) {
-            pc = ((intptr_t*)(stack->value - sizeof(intptr_t)))[0];
-            n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
-        }
-#endif
+
 #endif
 
     return FRAME_STEP(frame);
@@ -128,7 +123,7 @@
                                      int max_depth, int depth, intptr_t pc)
 {
     while (depth < max_depth && frame) {
-        frame = _write_python_stack_entry(frame, result, &depth, pc);
+        frame = _write_python_stack_entry(frame, result, &depth);
     }
     return depth;
 }
@@ -147,6 +142,17 @@
 
 int vmp_walk_and_record_stack(PY_STACK_FRAME_T *frame, void ** result,
                               int max_depth, int native_skip, intptr_t pc) {
+
+//#ifdef PYPY_JIT_CODEMAP
+//    intptr_t codemap_addr;
+//    if (pypy_find_codemap_at_addr((intptr_t)pc, &codemap_addr)) {
+//        // the bottom part is jitted, means we can fill up the first part
+//        // from the JIT
+//        depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
+//        frame = FRAME_STEP(frame); // skip the first item as it contains garbage
+//    }
+//#endif
+
     // called in signal handler
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
     intptr_t func_addr;
@@ -173,8 +179,8 @@
         native_skip--;
     }
 
+    int depth = 0;
     PY_STACK_FRAME_T * top_most_frame = frame;
-    int depth = 0;
     while (depth < max_depth) {
         unw_get_proc_info(&cursor, &pip);
 
@@ -205,7 +211,7 @@
                 if (top_most_frame == NULL) {
                     break;
                 }
-                top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth, pc);
+                top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth);
             }
         } else if (vmp_ignore_ip((intptr_t)func_addr)) {
             // this is an instruction pointer that should be ignored,
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -40,6 +40,8 @@
 #define PY_EVAL_RETURN_T void
 #define PY_THREAD_STATE_T void
 #define FRAME_STEP(f) f->next
+#define FRAME_CODE(f) f->
+PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag);
 #else
 // for cpython
 #include "_vmprof.h"
@@ -49,4 +51,8 @@
 #define PY_EVAL_RETURN_T PyObject
 #define PY_THREAD_STATE_T PyThreadState
 #define FRAME_STEP(f) f->f_back
+#define FRAME_CODE(f) f->f_code
+PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag);
 #endif
+
+

From pypy.commits at gmail.com  Tue Feb  7 03:57:42 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 07 Feb 2017 00:57:42 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: renamed stack.c to vmp_stack.c
 (name clash), ztranslation test passes now
Message-ID: <58998c06.4a86df0a.244b5.145d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89991:fc31028c801e
Date: 2017-02-07 09:57 +0100
http://bitbucket.org/pypy/pypy/changeset/fc31028c801e/

Log:	renamed stack.c to vmp_stack.c (name clash), ztranslation test
	passes now

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -53,7 +53,7 @@
         SHARED.join('compat.c'),
         SHARED.join('machine.c'),
         SHARED.join('symboltable.c'),
-        SHARED.join('stack.c'),
+        SHARED.join('vmp_stack.c'),
         # udis86
         SHARED.join('libudis86/decode.c'),
         SHARED.join('libudis86/itab.c'),
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -21,7 +21,7 @@
 #else
 #include "vmprof_main_win32.h"
 #endif
-#include "stack.h"
+#include "vmp_stack.h"
 
 static destructor Original_code_dealloc = 0;
 PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
diff --git a/rpython/rlib/rvmprof/src/shared/stack.c b/rpython/rlib/rvmprof/src/shared/stack.c
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/stack.c
+++ /dev/null
@@ -1,499 +0,0 @@
-#include "stack.h"
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <stddef.h>
-#include <assert.h>
-
-#include "vmprof.h"
-#include "compat.h"
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-#define UNW_LOCAL_ONLY
-#include <libunwind.h>
-#  ifdef X86_64
-#    define REG_RBX UNW_X86_64_RBX
-#  elif defined(X86_32)
-#    define REG_RBX UNW_X86_EDI
-#  endif
-#endif
-
-#ifdef __APPLE__
-#include <mach/mach.h>
-#include <mach/mach_vm.h>
-#include <mach/message.h>
-#include <mach/kern_return.h>
-#include <mach/task_info.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <dlfcn.h>
-#elif defined(__unix__)
-#include <dlfcn.h>
-#endif
-
-#ifdef PY_TEST
-// for testing only!
-PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag) { return NULL; }
-#endif
-
-static int vmp_native_traces_enabled = 0;
-static intptr_t *vmp_ranges = NULL;
-static ssize_t vmp_range_count = 0;
-static int _vmp_profiles_lines = 0;
-
-void vmp_profile_lines(int lines) {
-    _vmp_profiles_lines = lines;
-}
-int vmp_profiles_python_lines(void) {
-    return _vmp_profiles_lines;
-}
-
-static PY_STACK_FRAME_T * _write_python_stack_entry(PY_STACK_FRAME_T * frame, void ** result, int * depth)
-{
-    int len;
-    int addr;
-    int j;
-    long line;
-    char *lnotab;
-
-#ifndef RPYTHON_VMPROF // pypy does not support line profiling
-    if (vmp_profiles_python_lines()) {
-        // In the line profiling mode we save a line number for every frame.
-        // Actual line number isn't stored in the frame directly (f_lineno
-        // points to the beginning of the frame), so we need to compute it
-        // from f_lasti and f_code->co_lnotab. Here is explained what co_lnotab
-        // is:
-        // https://svn.python.org/projects/python/trunk/Objects/lnotab_notes.txt
-
-        // NOTE: the profiling overhead can be reduced by storing co_lnotab in the dump and
-        // moving this computation to the reader instead of doing it here.
-        lnotab = PyStr_AS_STRING(frame->f_code->co_lnotab);
-
-        if (lnotab != NULL) {
-            line = (long)frame->f_lineno;
-            addr = 0;
-
-            len = (int)PyStr_GET_SIZE(frame->f_code->co_lnotab);
-
-            for (j = 0; j < len; j += 2) {
-                addr += lnotab[j];
-                if (addr > frame->f_lasti) {
-                    break;
-                }
-                line += lnotab[j+1];
-            }
-            result[*depth] = (void*) line;
-            *depth = *depth + 1;
-        } else {
-            result[*depth] = (void*) 0;
-            *depth = *depth + 1;
-        }
-    }
-    result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
-    *depth = *depth + 1;
-#else
-    //result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
-    //*depth = *depth + 1;
-
-    if (frame->kind == VMPROF_CODE_TAG) {
-        int n = *depth;
-        result[n++] = (void*)frame->kind;
-        result[n++] = (void*)frame->value;
-        *depth = n;
-    }
-#ifdef PYPY_JIT_CODEMAP
-    else if (frame->kind == VMPROF_JITTED_TAG) {
-        intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
-        n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
-    }
-#endif
-
-#endif
-
-    return FRAME_STEP(frame);
-}
-
-int vmp_walk_and_record_python_stack_only(PY_STACK_FRAME_T *frame, void ** result,
-                                     int max_depth, int depth, intptr_t pc)
-{
-    while (depth < max_depth && frame) {
-        frame = _write_python_stack_entry(frame, result, &depth);
-    }
-    return depth;
-}
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-int _write_native_stack(void* addr, void ** result, int depth) {
-    if (vmp_profiles_python_lines()) {
-        // even if we do not log a python stack frame,
-        // we must keep the profile readable
-        result[depth++] = 0;
-    }
-    result[depth++] = addr;
-    return depth;
-}
-#endif
-
-int vmp_walk_and_record_stack(PY_STACK_FRAME_T *frame, void ** result,
-                              int max_depth, int native_skip, intptr_t pc) {
-
-//#ifdef PYPY_JIT_CODEMAP
-//    intptr_t codemap_addr;
-//    if (pypy_find_codemap_at_addr((intptr_t)pc, &codemap_addr)) {
-//        // the bottom part is jitted, means we can fill up the first part
-//        // from the JIT
-//        depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
-//        frame = FRAME_STEP(frame); // skip the first item as it contains garbage
-//    }
-//#endif
-
-    // called in signal handler
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    intptr_t func_addr;
-    unw_cursor_t cursor;
-    unw_context_t uc;
-    unw_proc_info_t pip;
-
-    if (!vmp_native_enabled()) {
-        return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
-    }
-
-    unw_getcontext(&uc);
-    int ret = unw_init_local(&cursor, &uc);
-    if (ret < 0) {
-        // could not initialize lib unwind cursor and context
-        return -1;
-    }
-
-    while (native_skip > 0) {
-        int err = unw_step(&cursor);
-        if (err <= 0) {
-            return 0;
-        }
-        native_skip--;
-    }
-
-    int depth = 0;
-    PY_STACK_FRAME_T * top_most_frame = frame;
-    while (depth < max_depth) {
-        unw_get_proc_info(&cursor, &pip);
-
-        func_addr = pip.start_ip;
-        //if (func_addr == 0) {
-        //    unw_word_t rip = 0;
-        //    if (unw_get_reg(&cursor, UNW_REG_IP, &rip) < 0) {
-        //        printf("failed failed failed\n");
-        //    }
-        //    func_addr = rip;
-        //    printf("func_addr is 0, now %p\n", rip);
-        //}
-
-
-        if ((void*)pip.start_ip == (void*)vmprof_eval) {
-            // yes we found one stack entry of the python frames!
-            unw_word_t rbx = 0;
-            if (unw_get_reg(&cursor, REG_RBX, &rbx) < 0) {
-                break;
-            }
-            if (rbx != (unw_word_t)top_most_frame) {
-                // uh we are screwed! the ip indicates we are have context
-                // to a PyEval_EvalFrameEx function, but when we tried to retrieve
-                // the stack located py frame it has a different address than the
-                // current top_most_frame
-                return 0;
-            } else {
-                if (top_most_frame == NULL) {
-                    break;
-                }
-                top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth);
-            }
-        } else if (vmp_ignore_ip((intptr_t)func_addr)) {
-            // this is an instruction pointer that should be ignored,
-            // (that is any function name in the mapping range of
-            //  cpython, but of course not extenstions in site-packages))
-            //printf("ignoring %s\n", info.dli_sname);
-        } else {
-            // mark native routines with the first bit set,
-            // this is possible because compiler align to 8 bytes.
-            //
-            depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
-        }
-
-        int err = unw_step(&cursor);
-        if (err <= 0) {
-            // on mac this breaks on Py_Main?
-            break;
-        }
-    }
-
-    if (top_most_frame == NULL) {
-        return depth;
-    }
-    // Whenever the trampoline is inserted, there might be a view python
-    // stack levels that do not have the trampoline!
-    // they should not be consumed, because the let native symbols flow forward.
-    return depth; //vmp_walk_and_record_python_stack_only(top_most_frame, result, max_depth, depth);
-#else
-    return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
-#endif
-}
-
-int vmp_native_enabled(void) {
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    return vmp_native_traces_enabled;
-#else
-    return 0;
-#endif
-}
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-int _ignore_symbols_from_path(const char * name) {
-    // which symbols should not be considered while walking
-    // the native stack?
-    if (strstr(name, "python") != NULL &&
-#ifdef __unix__
-        strstr(name, ".so\n") == NULL
-#elif defined(__APPLE__)
-        strstr(name, ".so") == NULL
-#endif
-       ) {
-        return 1;
-    }
-    return 0;
-}
-
-int _reset_vmp_ranges(void) {
-    // initially 10 (start, stop) entries!
-    int max_count = 10;
-    vmp_range_count = 0;
-    if (vmp_ranges != NULL) { free(vmp_ranges); }
-    vmp_ranges = malloc(max_count * sizeof(intptr_t));
-    return max_count;
-}
-
-
-int _resize_ranges(intptr_t ** cursor, int max_count) {
-    ptrdiff_t diff = (*cursor - vmp_ranges);
-    if (diff + 2 > max_count) {
-        max_count *= 2;
-        vmp_ranges = realloc(vmp_ranges, max_count*sizeof(intptr_t));
-        *cursor = vmp_ranges + diff;
-    }
-    return max_count;
-}
-
-intptr_t * _add_to_range(intptr_t * cursor, intptr_t start, intptr_t end) {
-    if (cursor[0] == start) {
-        // the last range is extended, this reduces the entry count
-        // which makes the querying faster
-        cursor[0] = end;
-    } else {
-        if (cursor != vmp_ranges) {
-            // not pointing to the first entry
-            cursor++;
-        }
-        cursor[0] = start;
-        cursor[1] = end;
-        vmp_range_count += 2;
-        cursor++;
-    }
-    return cursor;
-}
-
-#ifdef __unix__
-int vmp_read_vmaps(const char * fname) {
-
-    FILE * fd = fopen(fname, "rb");
-    if (fd == NULL) {
-        return 0;
-    }
-    char * saveptr;
-    char * line = NULL;
-    char * he = NULL;
-    char * name;
-    char *start_hex = NULL, *end_hex = NULL;
-    size_t n = 0;
-    ssize_t size;
-    intptr_t start, end;
-
-    // assumptions to be verified:
-    // 1) /proc/self/maps is ordered ascending by start address
-    // 2) libraries that contain the name 'python' are considered
-    //    candidates in the mapping to be ignored
-    // 3) libraries containing site-packages are not considered
-    //    candidates
-
-    int max_count = _reset_vmp_ranges();
-    intptr_t * cursor = vmp_ranges;
-    cursor[0] = -1;
-    while ((size = getline(&line, &n, fd)) >= 0) {
-        assert(line != NULL);
-        start_hex = strtok_r(line, "-", &saveptr);
-        if (start_hex == NULL) { continue; }
-        start = strtoll(start_hex, &he, 16);
-        end_hex = strtok_r(NULL, " ", &saveptr);
-        if (end_hex == NULL) { continue; }
-        end = strtoll(end_hex, &he, 16);
-        // skip over flags, ...
-        strtok_r(NULL, " ", &saveptr);
-        strtok_r(NULL, " ", &saveptr);
-        strtok_r(NULL, " ", &saveptr);
-        strtok_r(NULL, " ", &saveptr);
-
-        name = saveptr;
-        if (_ignore_symbols_from_path(name)) {
-            max_count = _resize_ranges(&cursor, max_count);
-            cursor = _add_to_range(cursor, start, end);
-        }
-        free(line);
-        line = NULL;
-        n = 0;
-    }
-
-    fclose(fd);
-    return 1;
-}
-#endif
-
-#ifdef __APPLE__
-int vmp_read_vmaps(const char * fname) {
-    kern_return_t kr;
-    task_t task;
-    mach_vm_address_t addr;
-    mach_vm_size_t vmsize;
-    vm_region_top_info_data_t topinfo;
-    mach_msg_type_number_t count;
-    memory_object_name_t obj;
-    int ret = 0;
-    pid_t pid;
-
-    pid = getpid();
-    kr = task_for_pid(mach_task_self(), pid, &task);
-    if (kr != KERN_SUCCESS) {
-        goto teardown;
-    }
-
-    addr = 0;
-    int max_count = _reset_vmp_ranges();
-    intptr_t * cursor = vmp_ranges;
-    cursor[0] = -1;
-
-    do {
-        // extract the top info using vm_region
-        count = VM_REGION_TOP_INFO_COUNT;
-        vmsize = 0;
-        kr = mach_vm_region(task, &addr, &vmsize, VM_REGION_TOP_INFO,
-                          (vm_region_info_t)&topinfo, &count, &obj);
-        if (kr == KERN_SUCCESS) {
-            vm_address_t start = addr, end = addr + vmsize;
-            // dladdr now gives the path of the shared object
-            Dl_info info;
-            if (dladdr((const void*)start, &info) == 0) {
-                // could not find image containing start
-                addr += vmsize;
-                continue;
-            }
-            if (_ignore_symbols_from_path(info.dli_fname)) {
-                // realloc if the chunk is to small
-                max_count = _resize_ranges(&cursor, max_count);
-                cursor = _add_to_range(cursor, start, end);
-            }
-            addr = addr + vmsize;
-        } else if (kr != KERN_INVALID_ADDRESS) {
-            goto teardown;
-        }
-    } while (kr == KERN_SUCCESS);
-
-    ret = 1;
-
-teardown:
-    if (task != MACH_PORT_NULL) {
-        mach_port_deallocate(mach_task_self(), task);
-    }
-    return ret;
-}
-#endif
-
-int vmp_native_enable(void) {
-    vmp_native_traces_enabled = 1;
-
-#if defined(__unix__)
-    return vmp_read_vmaps("/proc/self/maps");
-#elif defined(__APPLE__)
-    return vmp_read_vmaps(NULL);
-#endif
-}
-
-void vmp_native_disable(void) {
-    vmp_native_traces_enabled = 0;
-    if (vmp_ranges != NULL) {
-        free(vmp_ranges);
-        vmp_ranges = NULL;
-    }
-    vmp_range_count = 0;
-}
-
-int vmp_ignore_ip(intptr_t ip) {
-    int i = vmp_binary_search_ranges(ip, vmp_ranges, vmp_range_count);
-    if (i == -1) {
-        return 0;
-    }
-
-    assert((i & 1) == 0 && "returned index MUST be even");
-
-    intptr_t v = vmp_ranges[i];
-    intptr_t v2 = vmp_ranges[i+1];
-    return v <= ip && ip <= v2;
-}
-
-int vmp_binary_search_ranges(intptr_t ip, intptr_t * l, int count) {
-    intptr_t * r = l + count;
-    intptr_t * ol = l;
-    intptr_t * or = r-1;
-    while (1) {
-        ptrdiff_t i = (r-l)/2;
-        if (i == 0) {
-            if (l == ol && *l > ip) {
-                // at the start
-                return -1;
-            } else if (l == or && *l < ip) {
-                // at the end
-                return -1;
-            } else {
-                // we found the lower bound
-                i = l - ol;
-                if ((i & 1) == 1) {
-                    return i-1;
-                }
-                return i;
-            }
-        }
-        intptr_t * m = l + i;
-        if (ip < *m) {
-            r = m;
-        } else {
-            l = m;
-        }
-    }
-    return -1;
-}
-
-int vmp_ignore_symbol_count(void) {
-    return vmp_range_count;
-}
-
-intptr_t * vmp_ignore_symbols(void) {
-    return vmp_ranges;
-}
-
-void vmp_set_ignore_symbols(intptr_t * symbols, int count) {
-    vmp_ranges = symbols;
-    vmp_range_count = count;
-}
-#endif
diff --git a/rpython/rlib/rvmprof/src/shared/stack.h b/rpython/rlib/rvmprof/src/shared/stack.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/shared/stack.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#pragma once
-
-#include "vmprof.h"
-
-int vmp_walk_and_record_stack(PY_STACK_FRAME_T * frame, void **data,
-                              int max_depth, int native_skip, intptr_t pc);
-
-int vmp_native_enabled(void);
-int vmp_native_enable(void);
-int vmp_ignore_ip(intptr_t ip);
-int vmp_binary_search_ranges(intptr_t ip, intptr_t * l, int count);
-int vmp_native_symbols_read(void);
-void vmp_profile_lines(int lines);
-int vmp_profiles_python_lines(void);
-
-int vmp_ignore_symbol_count(void);
-intptr_t * vmp_ignore_symbols(void);
-void vmp_set_ignore_symbols(intptr_t * symbols, int count);
-void vmp_native_disable(void);
-
-#ifdef __unix__
-int vmp_read_vmaps(const char * fname);
-#endif
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -35,7 +35,7 @@
 
 #include "vmprof.h"
 
-#include "stack.h"
+#include "vmp_stack.h"
 #include "vmprof_getpc.h"
 #include "vmprof_mt.h"
 #include "vmprof_common.h"
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c
@@ -1,7 +1,7 @@
 
 #include "windows.h"
 #include "compat.h"
-#include "stack.h"
+#include "vmp_stack.h"
 
 HANDLE write_mutex;
 
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
@@ -2,7 +2,7 @@
 
 #include "windows.h"
 #include "compat.h"
-#include "stack.h"
+#include "vmp_stack.h"
 
 HANDLE write_mutex;
 

From pypy.commits at gmail.com  Tue Feb  7 04:28:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 01:28:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix for direct tests
Message-ID: <58999327.c53f1c0a.e9477.bd37@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89992:b24a3e616dc8
Date: 2017-02-07 10:27 +0100
http://bitbucket.org/pypy/pypy/changeset/b24a3e616dc8/

Log:	fix for direct tests

diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -2,7 +2,8 @@
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
 
-from reprlib import recursive_repr as _recursive_repr
+try: from reprlib import recursive_repr as _recursive_repr
+except ImportError: _recursive_repr = lambda: (lambda f: f)
 
 
 sentinel = object()

From pypy.commits at gmail.com  Tue Feb  7 04:41:48 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 07 Feb 2017 01:41:48 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: add doc string to jit.oopspec,
 parameterize native on rvmprof.enable
Message-ID: <5899965c.0a081c0a.d0dc.c244@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89993:a947b5fb780a
Date: 2017-02-07 10:21 +0100
http://bitbucket.org/pypy/pypy/changeset/a947b5fb780a/

Log:	add doc string to jit.oopspec, parameterize native on rvmprof.enable

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -244,6 +244,10 @@
     return inner
 
 def oopspec(spec):
+    """ The JIT compiler won't look inside this decorated function,
+        but instead during translation, rewrites it according to the handler in
+        rpython/jit/codewriter/jtransform.py.
+    """
     def decorator(func):
         func.oopspec = spec
         return func
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -122,7 +122,7 @@
         self._gather_all_code_objs = gather_all_code_objs
 
     @jit.dont_look_inside
-    def enable(self, fileno, interval, memory=0):
+    def enable(self, fileno, interval, memory=0, native=0):
         """Enable vmprof.  Writes go to the given 'fileno'.
         The sampling interval is given by 'interval' as a number of
         seconds, as a float which must be smaller than 1.0.
@@ -132,8 +132,7 @@
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
 
-        lines = 0
-        native = 0
+        lines = 0 # not supported on PyPy currently
 
         p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native)
         if p_error:

From pypy.commits at gmail.com  Tue Feb  7 04:42:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 01:42:09 -0800 (PST)
Subject: [pypy-commit] pypy default: Only ignore "*_cffi.c", not all .c files
Message-ID: <58999671.d5941c0a.33214.c848@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89994:45712b335ecf
Date: 2017-02-07 10:41 +0100
http://bitbucket.org/pypy/pypy/changeset/45712b335ecf/

Log:	Only ignore "*_cffi.c", not all .c files

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -179,7 +179,7 @@
     shutil.copytree(str(basedir.join('lib_pypy')),
                     str(pypydir.join('lib_pypy')),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
-                                           '*.c', '*.o'))
+                                           '*_cffi.c', '*.o'))
     for file in ['README.rst',]:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     for file in ['_testcapimodule.c', '_ctypes_test.c']:

From pypy.commits at gmail.com  Tue Feb  7 04:43:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 01:43:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <589996a6.b296df0a.558fa.2242@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89995:7cab0b276e26
Date: 2017-02-07 10:42 +0100
http://bitbucket.org/pypy/pypy/changeset/7cab0b276e26/

Log:	hg merge default

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -283,21 +283,15 @@
         w_value = self.getitem(w_key)
         return w_value if w_value is not None else w_default
 
-    @unwrap_spec(defaults_w='args_w')
-    def descr_pop(self, space, w_key, defaults_w):
+    def descr_pop(self, space, w_key, w_default=None):
         """D.pop(k[,d]) -> v, remove specified key and return the
         corresponding value\nIf key is not found, d is returned if given,
         otherwise KeyError is raised
         """
-        len_defaults = len(defaults_w)
-        if len_defaults > 1:
-            raise oefmt(space.w_TypeError,
-                        "pop expected at most 2 arguments, got %d",
-                        1 + len_defaults)
         w_item = self.getitem(w_key)
         if w_item is None:
-            if len_defaults > 0:
-                return defaults_w[0]
+            if w_default is not None:
+                return w_default
             else:
                 space.raise_key_error(w_key)
         else:
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -179,7 +179,7 @@
     shutil.copytree(str(basedir.join('lib_pypy')),
                     str(pypydir.join('lib_pypy')),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
-                                           '*.c', '*.o'))
+                                           '*_cffi.c', '*.o'))
     for file in ['README.rst',]:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     for file in ['_testcapimodule.c', '_ctypes_test.c']:

From pypy.commits at gmail.com  Tue Feb  7 05:54:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 02:54:15 -0800 (PST)
Subject: [pypy-commit] pypy default: Add some debugging logic
Message-ID: <5899a757.c80e1c0a.668c9.e9dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89996:3195205d1faa
Date: 2017-02-07 11:53 +0100
http://bitbucket.org/pypy/pypy/changeset/3195205d1faa/

Log:	Add some debugging logic

diff --git a/rpython/translator/c/src/stacklet/stacklet.c b/rpython/translator/c/src/stacklet/stacklet.c
--- a/rpython/translator/c/src/stacklet/stacklet.c
+++ b/rpython/translator/c/src/stacklet/stacklet.c
@@ -34,7 +34,7 @@
 /************************************************************/
 
 struct stacklet_s {
-    /* The portion of the real stack claimed by this paused tealet. */
+    /* The portion of the real stack claimed by this paused stacklet. */
     char *stack_start;                /* the "near" end of the stack */
     char *stack_stop;                 /* the "far" end of the stack */
 
@@ -69,6 +69,14 @@
     struct stacklet_s *g_target;
 };
 
+static void check_valid(struct stacklet_s *g)
+{
+    if (g->stack_saved < 0) {
+        fprintf(stderr, "FATAL: stacklet: memory corruption\n");
+        abort();
+    }
+}
+
 /***************************************************************/
 
 static void g_save(struct stacklet_s* g, char* stop
@@ -96,6 +104,7 @@
      */
     ptrdiff_t sz1 = g->stack_saved;
     ptrdiff_t sz2 = stop - g->stack_start;
+    check_valid(g);
     assert(stop <= g->stack_stop);
 
     if (sz2 > sz1) {
@@ -146,11 +155,13 @@
 {
     struct stacklet_s *current = thrd->g_stack_chain_head;
     char *target_stop = g_target->stack_stop;
+    check_valid(g_target);
 
-    /* save and unlink tealets that are completely within
+    /* save and unlink stacklets that are completely within
        the area to clear. */
     while (current != NULL && current->stack_stop <= target_stop) {
         struct stacklet_s *prev = current->stack_prev;
+        check_valid(current);
         current->stack_prev = NULL;
         if (current != g_target) {
             /* don't bother saving away g_target, because
@@ -222,6 +233,7 @@
     struct stacklet_thread_s *thrd = (struct stacklet_thread_s *)rawthrd;
     struct stacklet_s *g = thrd->g_target;
     ptrdiff_t stack_saved = g->stack_saved;
+    check_valid(g);
 
     assert(new_stack_pointer == g->stack_start);
 #if STACK_DIRECTION == 0
@@ -230,6 +242,7 @@
     memcpy(g->stack_start - stack_saved, g+1, stack_saved);
 #endif
     thrd->g_current_stack_stop = g->stack_stop;
+    g->stack_saved = -13;   /* debugging */
     free(g);
     return EMPTY_STACKLET_HANDLE;
 }
@@ -250,6 +263,7 @@
         result = run(thrd->g_source, run_arg);
 
         /* Then switch to 'result'. */
+        check_valid(result);
         thrd->g_target = result;
         _stacklet_switchstack(g_destroy_state, g_restore_state, thrd);
 
@@ -300,6 +314,7 @@
 {
     long stackmarker;
     stacklet_thread_handle thrd = target->stack_thrd;
+    check_valid(target);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
@@ -310,6 +325,7 @@
 
 void stacklet_destroy(stacklet_handle target)
 {
+    check_valid(target);
     if (target->stack_prev != NULL) {
         /* 'target' appears to be in the chained list 'unsaved_stack',
            so remove it from there.  Note that if 'thrd' was already
@@ -319,12 +335,15 @@
            we don't even read 'stack_thrd', already deallocated. */
         stacklet_thread_handle thrd = target->stack_thrd;
         struct stacklet_s **pp = &thrd->g_stack_chain_head;
-        for (; *pp != NULL; pp = &(*pp)->stack_prev)
+        for (; *pp != NULL; pp = &(*pp)->stack_prev) {
+            check_valid(*pp);
             if (*pp == target) {
                 *pp = target->stack_prev;
                 break;
             }
+        }
     }
+    target->stack_saved = -11;   /* debugging */
     free(target);
 }
 
@@ -334,6 +353,7 @@
   long delta;
   if (context == NULL)
     return ptr;
+  check_valid(context);
   delta = p - context->stack_start;
   if (((unsigned long)delta) < ((unsigned long)context->stack_saved)) {
       /* a pointer to a saved away word */

From pypy.commits at gmail.com  Tue Feb  7 06:02:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 03:02:01 -0800 (PST)
Subject: [pypy-commit] pypy default: fix,
 and also replace asserts with checks that are always enabled
Message-ID: <5899a929.ad8ddf0a.8cbd0.46bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r89997:4fcac853f676
Date: 2017-02-07 12:01 +0100
http://bitbucket.org/pypy/pypy/changeset/4fcac853f676/

Log:	fix, and also replace asserts with checks that are always enabled

diff --git a/rpython/translator/c/src/stacklet/stacklet.c b/rpython/translator/c/src/stacklet/stacklet.c
--- a/rpython/translator/c/src/stacklet/stacklet.c
+++ b/rpython/translator/c/src/stacklet/stacklet.c
@@ -5,8 +5,8 @@
 #include "src/stacklet/stacklet.h"
 
 #include <stddef.h>
-#include <assert.h>
 #include <string.h>
+#include <stdio.h>
 
 /************************************************************
  * platform specific code
@@ -69,12 +69,17 @@
     struct stacklet_s *g_target;
 };
 
+#define _check(x)  do { if (!(x)) _check_failed(#x); } while (0)
+
+static void _check_failed(const char *check)
+{
+    fprintf(stderr, "FATAL: stacklet: %s failed\n", check);
+    abort();
+}
+
 static void check_valid(struct stacklet_s *g)
 {
-    if (g->stack_saved < 0) {
-        fprintf(stderr, "FATAL: stacklet: memory corruption\n");
-        abort();
-    }
+    _check(g->stack_saved >= 0);
 }
 
 /***************************************************************/
@@ -105,7 +110,7 @@
     ptrdiff_t sz1 = g->stack_saved;
     ptrdiff_t sz2 = stop - g->stack_start;
     check_valid(g);
-    assert(stop <= g->stack_stop);
+    _check(stop <= g->stack_stop);
 
     if (sz2 > sz1) {
         char *c = (char *)(g + 1);
@@ -235,7 +240,7 @@
     ptrdiff_t stack_saved = g->stack_saved;
     check_valid(g);
 
-    assert(new_stack_pointer == g->stack_start);
+    _check(new_stack_pointer == g->stack_start);
 #if STACK_DIRECTION == 0
     memcpy(g->stack_start, g+1, stack_saved);
 #else
@@ -267,7 +272,7 @@
         thrd->g_target = result;
         _stacklet_switchstack(g_destroy_state, g_restore_state, thrd);
 
-        assert(!"stacklet: we should not return here");
+        _check_failed("we should not return here");
         abort();
     }
     /* The second time it returns. */
@@ -301,7 +306,7 @@
                              stacklet_run_fn run, void *run_arg)
 {
     long stackmarker;
-    assert((char *)NULL < (char *)&stackmarker);
+    _check((char *)NULL < (char *)&stackmarker);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
@@ -365,8 +370,8 @@
       /* out-of-stack pointer!  it's only ok if we are the main stacklet
          and we are reading past the end, because the main stacklet's
          stack stop is not exactly known. */
-      assert(delta >= 0);
-      assert(((long)context->stack_stop) & 1);
+      _check(delta >= 0);
+      _check(((long)context->stack_stop) & 1);
   }
   return ptr;
 }

From pypy.commits at gmail.com  Tue Feb  7 08:31:09 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 07 Feb 2017 05:31:09 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: passing some more vmprof tests
Message-ID: <5899cc1d.9d711c0a.875ba.311b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r89998:dc4c3b35b94c
Date: 2017-02-07 14:29 +0100
http://bitbucket.org/pypy/pypy/changeset/dc4c3b35b94c/

Log:	passing some more vmprof tests

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -18,7 +18,7 @@
 UDIS86 = SHARED.join('libudis86')
 BACKTRACE = SHARED.join('libbacktrace')
 
-compile_extra = ['-DRPYTHON_LL2CTYPES','-DRPYTHON_VMPROF']
+compile_extra = ['-DRPYTHON_VMPROF', '-g', '-O1']
 if sys.platform.startswith('linux'):
     separate_module_files = [
        BACKTRACE.join('backtrace.c'),
@@ -66,14 +66,17 @@
 
 
 def setup():
-    platform.verify_eci(ExternalCompilationInfo(**eci_kwds))
+    eci_kwds['compile_extra'].append('-DRPYTHON_LL2CTYPES')
+    platform.verify_eci(ExternalCompilationInfo(
+                        **eci_kwds))
 
     eci = global_eci
     vmprof_init = rffi.llexternal("vmprof_init",
                                   [rffi.INT, rffi.DOUBLE, rffi.INT, rffi.INT,
                                    rffi.CCHARP, rffi.INT],
                                   rffi.CCHARP, compilation_info=eci)
-    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT], rffi.INT,
+    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT, rffi.INT],
+                                    rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
     vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -10,6 +10,8 @@
 
 MAX_FUNC_NAME = 1023
 
+PLAT_WINDOWS = sys.platform == 'win32'
+
 # ____________________________________________________________
 
 # keep in sync with vmprof_stack.h
@@ -132,6 +134,8 @@
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
 
+        if PLAT_WINDOWS:
+            native = 0 # force disabled on Windows
         lines = 0 # not supported on PyPy currently
 
         p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native)
@@ -139,7 +143,7 @@
             raise VMProfError(rffi.charp2str(p_error))
 
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable(memory)
+        res = self.cintf.vmprof_enable(memory, native)
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -3,8 +3,6 @@
 #ifdef RPYTHON_LL2CTYPES
    /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
 
-static volatile int is_enabled = 0;
-
 #else
 #  include "common_header.h"
 #  include "structdef.h"
@@ -47,3 +45,17 @@
     return NULL; // TODO _default_eval_loop(f, throwflag);
 }
 #endif
+
+void dump_native_symbols(int fileno)
+{
+// TODO    PyObject * mod = NULL;
+// TODO
+// TODO    mod = PyImport_ImportModuleNoBlock("vmprof");
+// TODO    if (mod == NULL)
+// TODO        goto error;
+// TODO
+// TODO    PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
+// TODO
+// TODOerror:
+// TODO    Py_XDECREF(mod);
+}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -25,7 +25,7 @@
 RPY_EXTERN char *vmprof_init(int fd, double interval, int memory,
                      int lines, const char *interp_name, int native);
 RPY_EXTERN void vmprof_ignore_signals(int);
-RPY_EXTERN int vmprof_enable(int memory);
+RPY_EXTERN int vmprof_enable(int memory, int native);
 RPY_EXTERN int vmprof_disable(void);
 RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
 RPY_EXTERN void* vmprof_stack_new(void);
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -12,6 +12,8 @@
 #include "_vmprof.h"
 
 static volatile int is_enabled = 0;
+static destructor Original_code_dealloc = 0;
+static PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
 
 #if VMPROF_UNIX
 #include "trampoline.h"
@@ -23,9 +25,6 @@
 #endif
 #include "vmp_stack.h"
 
-static destructor Original_code_dealloc = 0;
-PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
-
 #ifdef VMPROF_UNIX
 #ifdef __clang__
 __attribute__((disable_tail_calls))
@@ -150,30 +149,6 @@
     Original_code_dealloc(co);
 }
 
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-static void init_cpyprof(int native)
-{
-    // skip this if native should not be enabled
-    if (!native) {
-        vmp_native_disable();
-        return;
-    }
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = vmprof_eval;
-    _default_eval_loop = _PyEval_EvalFrameDefault;
-#else
-    if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx,
-                vmprof_eval, (void*)&_default_eval_loop) == 0) {
-    } else {
-        fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n");
-        // TODO dump the first few bytes and tell them to create an issue!
-        exit(-1);
-    }
-#endif
-    vmp_native_enable();
-}
-#endif
 
 void dump_native_symbols(int fileno)
 {
@@ -189,22 +164,6 @@
     Py_XDECREF(mod);
 }
 
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-static void disable_cpyprof(void)
-{
-    vmp_native_disable();
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = _PyEval_EvalFrameDefault;
-#else
-    if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) {
-        fprintf(stderr, "FATAL: could not remove trampoline\n");
-        exit(-1);
-    }
-#endif
-    dump_native_symbols(vmp_profile_fileno());
-}
-#endif
 
 
 static PyObject *enable_vmprof(PyObject* self, PyObject *args)
@@ -229,10 +188,6 @@
 
     vmp_profile_lines(lines);
 
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    init_cpyprof(native);
-#endif
-
     if (!Original_code_dealloc) {
         Original_code_dealloc = PyCode_Type.tp_dealloc;
         PyCode_Type.tp_dealloc = &cpyprof_code_dealloc;
@@ -244,7 +199,7 @@
         return NULL;
     }
 
-    if (vmprof_enable(memory) < 0) {
+    if (vmprof_enable(memory, native) < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
     }
@@ -265,9 +220,6 @@
     is_enabled = 0;
     vmprof_ignore_signals(1);
     emit_all_code_objects();
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    disable_cpyprof();
-#endif
 
     if (vmprof_disable() < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
@@ -319,7 +271,7 @@
         vmprof_ignore_signals(0);
         return NULL;
     }
-    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, 0);
+    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, 0, 0);
 
     for (i = 0; i < entry_count; i++) {
         routine_ip = m[i];
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.h b/rpython/rlib/rvmprof/src/shared/_vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.h
@@ -8,6 +8,7 @@
 #else
 #include <inttypes.h>
 #include <stdint.h>
+#include <stddef.h>
 #endif
 
 /**
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -9,19 +9,7 @@
 
 #include <dlfcn.h>
 #ifdef VMPROF_LINUX
-#define _GNU_SOURCE 1
 #include <link.h>
-// copied from man page...
-typedef struct {
-   const char *dli_fname;  /* Pathname of shared object that
-                              contains address */
-   void       *dli_fbase;  /* Base address at which shared
-                              object is loaded */
-   const char *dli_sname;  /* Name of symbol whose definition
-                              overlaps addr */
-   void       *dli_saddr;  /* Exact address of symbol named
-                              in dli_sname */
-} Dl_info;
 #endif
 
 #ifdef _PY_TEST
@@ -232,7 +220,7 @@
         name[name_len-1] = 0;
     }
     lookup_vmprof_debug_info(name, info.dli_fbase, srcfile, srcfile_len, lineno);
-#elif defined(VMPROF_LINUX)
+#elif defined(__unix__)
     if (bstate == NULL) {
         bstate = backtrace_create_state (NULL, 1, backtrace_error_cb, NULL);
     }
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.h b/rpython/rlib/rvmprof/src/shared/symboltable.h
--- a/rpython/rlib/rvmprof/src/shared/symboltable.h
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#define _GNU_SOURCE 1
+
 /**
  * Extract all the known symbols from the current process and
  * log them to the file descriptor. To read them see binary.py funcs:
diff --git a/rpython/rlib/rvmprof/src/shared/trampoline.c b/rpython/rlib/rvmprof/src/shared/trampoline.c
--- a/rpython/rlib/rvmprof/src/shared/trampoline.c
+++ b/rpython/rlib/rvmprof/src/shared/trampoline.c
@@ -1,7 +1,7 @@
 #include "trampoline.h"
 
+#include "vmprof.h"
 #include "machine.h"
-#include "_vmprof.h"
 
 #define _GNU_SOURCE 1
 #include <string.h>
@@ -141,6 +141,9 @@
     while (bytes < needed_bytes) {
         unsigned int res = vmp_machine_code_instr_length(ptr);
         if (res == 0) {
+            fprintf(stderr, "could not determine length of instr for trampoline\n");
+            fprintf(stderr, " %x %x %x %x %x\n", ptr[0], ptr[1],
+                            ptr[2], ptr[3], ptr[4]);
             return 1;
         }
 #ifdef X86_32
@@ -180,6 +183,7 @@
     int pagesize;
 
     if (g_trampoline != NULL) {
+        fprintf(stderr, "trampoline already patched\n");
         return 0; // already patched
     }
 
@@ -195,11 +199,13 @@
     char * page = (char*)mmap(NULL, pagesize, PROT_READ|PROT_WRITE|PROT_EXEC,
                               MAP_ANON | MAP_PRIVATE, 0, 0);
     if (page == NULL) {
+        fprintf(stderr, "could not allocate page for trampoline\n");
         return -1;
     }
 
     char * a = (char*)callee_addr;
     if (_redirect_trampoline_and_back(a, page, vmprof_eval) != 0) {
+        fprintf(stderr, "could not redirect eval->vmprof_eval->trampoline->eval+off\n");
         return -1;
     }
 
@@ -224,33 +230,4 @@
 int vmp_unpatch_callee_trampoline(void * callee_addr)
 {
     return 0; // currently the trampoline is not removed
-
-    //if (!g_patched) {
-    //    return -1;
-    //}
-
-    //int result;
-    //int pagesize = sysconf(_SC_PAGESIZE);
-    //errno = 0;
-
-    //result = mprotect(PAGE_ALIGNED(callee_addr, pagesize), pagesize*2, PROT_READ|PROT_WRITE);
-    //if (result != 0) {
-    //    fprintf(stderr, "read|write protecting callee_addr\n");
-    //    return 1;
-    //}
-
-    //// copy back, assume everything is as if nothing ever happened!!
-    //(void)memcpy(callee_addr, g_trampoline, g_trampoline_length);
-
-    //result = mprotect(PAGE_ALIGNED(callee_addr, pagesize), pagesize*2, PROT_READ|PROT_EXEC);
-    //if (result != 0) {
-    //    fprintf(stderr, "read|exec protecting callee addr\n");
-    //    return 1;
-    //}
-
-    //munmap(g_trampoline, pagesize);
-    //g_trampoline = NULL;
-    //g_trampoline_length = 0;
-
-    //return 0;
 }
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -0,0 +1,499 @@
+#include "vmp_stack.h"
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <stddef.h>
+#include <assert.h>
+
+#include "vmprof.h"
+#include "compat.h"
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+#define UNW_LOCAL_ONLY
+#include <libunwind.h>
+#  ifdef X86_64
+#    define REG_RBX UNW_X86_64_RBX
+#  elif defined(X86_32)
+#    define REG_RBX UNW_X86_EDI
+#  endif
+#endif
+
+#ifdef __APPLE__
+#include <mach/mach.h>
+#include <mach/mach_vm.h>
+#include <mach/message.h>
+#include <mach/kern_return.h>
+#include <mach/task_info.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <dlfcn.h>
+#elif defined(__unix__)
+#include <dlfcn.h>
+#endif
+
+#ifdef PY_TEST
+// for testing only!
+PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag) { return NULL; }
+#endif
+
+static int vmp_native_traces_enabled = 0;
+static intptr_t *vmp_ranges = NULL;
+static ssize_t vmp_range_count = 0;
+static int _vmp_profiles_lines = 0;
+
+void vmp_profile_lines(int lines) {
+    _vmp_profiles_lines = lines;
+}
+int vmp_profiles_python_lines(void) {
+    return _vmp_profiles_lines;
+}
+
+static PY_STACK_FRAME_T * _write_python_stack_entry(PY_STACK_FRAME_T * frame, void ** result, int * depth)
+{
+    int len;
+    int addr;
+    int j;
+    long line;
+    char *lnotab;
+
+#ifndef RPYTHON_VMPROF // pypy does not support line profiling
+    if (vmp_profiles_python_lines()) {
+        // In the line profiling mode we save a line number for every frame.
+        // Actual line number isn't stored in the frame directly (f_lineno
+        // points to the beginning of the frame), so we need to compute it
+        // from f_lasti and f_code->co_lnotab. Here is explained what co_lnotab
+        // is:
+        // https://svn.python.org/projects/python/trunk/Objects/lnotab_notes.txt
+
+        // NOTE: the profiling overhead can be reduced by storing co_lnotab in the dump and
+        // moving this computation to the reader instead of doing it here.
+        lnotab = PyStr_AS_STRING(frame->f_code->co_lnotab);
+
+        if (lnotab != NULL) {
+            line = (long)frame->f_lineno;
+            addr = 0;
+
+            len = (int)PyStr_GET_SIZE(frame->f_code->co_lnotab);
+
+            for (j = 0; j < len; j += 2) {
+                addr += lnotab[j];
+                if (addr > frame->f_lasti) {
+                    break;
+                }
+                line += lnotab[j+1];
+            }
+            result[*depth] = (void*) line;
+            *depth = *depth + 1;
+        } else {
+            result[*depth] = (void*) 0;
+            *depth = *depth + 1;
+        }
+    }
+    result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
+    *depth = *depth + 1;
+#else
+    //result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
+    //*depth = *depth + 1;
+
+    if (frame->kind == VMPROF_CODE_TAG) {
+        int n = *depth;
+        result[n++] = (void*)frame->kind;
+        result[n++] = (void*)frame->value;
+        *depth = n;
+    }
+#ifdef PYPY_JIT_CODEMAP
+    else if (frame->kind == VMPROF_JITTED_TAG) {
+        intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
+        n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
+    }
+#endif
+
+#endif
+
+    return FRAME_STEP(frame);
+}
+
+int vmp_walk_and_record_python_stack_only(PY_STACK_FRAME_T *frame, void ** result,
+                                     int max_depth, int depth, intptr_t pc)
+{
+    while (depth < max_depth && frame) {
+        frame = _write_python_stack_entry(frame, result, &depth);
+    }
+    return depth;
+}
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+int _write_native_stack(void* addr, void ** result, int depth) {
+    if (vmp_profiles_python_lines()) {
+        // even if we do not log a python stack frame,
+        // we must keep the profile readable
+        result[depth++] = 0;
+    }
+    result[depth++] = addr;
+    return depth;
+}
+#endif
+
+int vmp_walk_and_record_stack(PY_STACK_FRAME_T *frame, void ** result,
+                              int max_depth, int native_skip, intptr_t pc) {
+
+//#ifdef PYPY_JIT_CODEMAP
+//    intptr_t codemap_addr;
+//    if (pypy_find_codemap_at_addr((intptr_t)pc, &codemap_addr)) {
+//        // the bottom part is jitted, means we can fill up the first part
+//        // from the JIT
+//        depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
+//        frame = FRAME_STEP(frame); // skip the first item as it contains garbage
+//    }
+//#endif
+
+    // called in signal handler
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    intptr_t func_addr;
+    unw_cursor_t cursor;
+    unw_context_t uc;
+    unw_proc_info_t pip;
+
+    if (!vmp_native_enabled()) {
+        return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
+    }
+
+    unw_getcontext(&uc);
+    int ret = unw_init_local(&cursor, &uc);
+    if (ret < 0) {
+        // could not initialize lib unwind cursor and context
+        return -1;
+    }
+
+    while (native_skip > 0) {
+        int err = unw_step(&cursor);
+        if (err <= 0) {
+            return 0;
+        }
+        native_skip--;
+    }
+
+    int depth = 0;
+    PY_STACK_FRAME_T * top_most_frame = frame;
+    while (depth < max_depth) {
+        unw_get_proc_info(&cursor, &pip);
+
+        func_addr = pip.start_ip;
+        //if (func_addr == 0) {
+        //    unw_word_t rip = 0;
+        //    if (unw_get_reg(&cursor, UNW_REG_IP, &rip) < 0) {
+        //        printf("failed failed failed\n");
+        //    }
+        //    func_addr = rip;
+        //    printf("func_addr is 0, now %p\n", rip);
+        //}
+
+
+        if ((void*)pip.start_ip == (void*)vmprof_eval) {
+            // yes we found one stack entry of the python frames!
+            unw_word_t rbx = 0;
+            if (unw_get_reg(&cursor, REG_RBX, &rbx) < 0) {
+                break;
+            }
+            if (rbx != (unw_word_t)top_most_frame) {
+                // uh we are screwed! the ip indicates we are have context
+                // to a PyEval_EvalFrameEx function, but when we tried to retrieve
+                // the stack located py frame it has a different address than the
+                // current top_most_frame
+                return 0;
+            } else {
+                if (top_most_frame == NULL) {
+                    break;
+                }
+                top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth);
+            }
+        } else if (vmp_ignore_ip((intptr_t)func_addr)) {
+            // this is an instruction pointer that should be ignored,
+            // (that is any function name in the mapping range of
+            //  cpython, but of course not extenstions in site-packages))
+            //printf("ignoring %s\n", info.dli_sname);
+        } else {
+            // mark native routines with the first bit set,
+            // this is possible because compiler align to 8 bytes.
+            //
+            depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
+        }
+
+        int err = unw_step(&cursor);
+        if (err <= 0) {
+            // on mac this breaks on Py_Main?
+            break;
+        }
+    }
+
+    if (top_most_frame == NULL) {
+        return depth;
+    }
+    // Whenever the trampoline is inserted, there might be a view python
+    // stack levels that do not have the trampoline!
+    // they should not be consumed, because the let native symbols flow forward.
+    return depth; //vmp_walk_and_record_python_stack_only(top_most_frame, result, max_depth, depth);
+#else
+    return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
+#endif
+}
+
+int vmp_native_enabled(void) {
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    return vmp_native_traces_enabled;
+#else
+    return 0;
+#endif
+}
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+int _ignore_symbols_from_path(const char * name) {
+    // which symbols should not be considered while walking
+    // the native stack?
+    if (strstr(name, "python") != NULL &&
+#ifdef __unix__
+        strstr(name, ".so\n") == NULL
+#elif defined(__APPLE__)
+        strstr(name, ".so") == NULL
+#endif
+       ) {
+        return 1;
+    }
+    return 0;
+}
+
+int _reset_vmp_ranges(void) {
+    // initially 10 (start, stop) entries!
+    int max_count = 10;
+    vmp_range_count = 0;
+    if (vmp_ranges != NULL) { free(vmp_ranges); }
+    vmp_ranges = malloc(max_count * sizeof(intptr_t));
+    return max_count;
+}
+
+
+int _resize_ranges(intptr_t ** cursor, int max_count) {
+    ptrdiff_t diff = (*cursor - vmp_ranges);
+    if (diff + 2 > max_count) {
+        max_count *= 2;
+        vmp_ranges = realloc(vmp_ranges, max_count*sizeof(intptr_t));
+        *cursor = vmp_ranges + diff;
+    }
+    return max_count;
+}
+
+intptr_t * _add_to_range(intptr_t * cursor, intptr_t start, intptr_t end) {
+    if (cursor[0] == start) {
+        // the last range is extended, this reduces the entry count
+        // which makes the querying faster
+        cursor[0] = end;
+    } else {
+        if (cursor != vmp_ranges) {
+            // not pointing to the first entry
+            cursor++;
+        }
+        cursor[0] = start;
+        cursor[1] = end;
+        vmp_range_count += 2;
+        cursor++;
+    }
+    return cursor;
+}
+
+#ifdef __unix__
+int vmp_read_vmaps(const char * fname) {
+
+    FILE * fd = fopen(fname, "rb");
+    if (fd == NULL) {
+        return 0;
+    }
+    char * saveptr;
+    char * line = NULL;
+    char * he = NULL;
+    char * name;
+    char *start_hex = NULL, *end_hex = NULL;
+    size_t n = 0;
+    ssize_t size;
+    intptr_t start, end;
+
+    // assumptions to be verified:
+    // 1) /proc/self/maps is ordered ascending by start address
+    // 2) libraries that contain the name 'python' are considered
+    //    candidates in the mapping to be ignored
+    // 3) libraries containing site-packages are not considered
+    //    candidates
+
+    int max_count = _reset_vmp_ranges();
+    intptr_t * cursor = vmp_ranges;
+    cursor[0] = -1;
+    while ((size = getline(&line, &n, fd)) >= 0) {
+        assert(line != NULL);
+        start_hex = strtok_r(line, "-", &saveptr);
+        if (start_hex == NULL) { continue; }
+        start = strtoll(start_hex, &he, 16);
+        end_hex = strtok_r(NULL, " ", &saveptr);
+        if (end_hex == NULL) { continue; }
+        end = strtoll(end_hex, &he, 16);
+        // skip over flags, ...
+        strtok_r(NULL, " ", &saveptr);
+        strtok_r(NULL, " ", &saveptr);
+        strtok_r(NULL, " ", &saveptr);
+        strtok_r(NULL, " ", &saveptr);
+
+        name = saveptr;
+        if (_ignore_symbols_from_path(name)) {
+            max_count = _resize_ranges(&cursor, max_count);
+            cursor = _add_to_range(cursor, start, end);
+        }
+        free(line);
+        line = NULL;
+        n = 0;
+    }
+
+    fclose(fd);
+    return 1;
+}
+#endif
+
+#ifdef __APPLE__
+int vmp_read_vmaps(const char * fname) {
+    kern_return_t kr;
+    task_t task;
+    mach_vm_address_t addr;
+    mach_vm_size_t vmsize;
+    vm_region_top_info_data_t topinfo;
+    mach_msg_type_number_t count;
+    memory_object_name_t obj;
+    int ret = 0;
+    pid_t pid;
+
+    pid = getpid();
+    kr = task_for_pid(mach_task_self(), pid, &task);
+    if (kr != KERN_SUCCESS) {
+        goto teardown;
+    }
+
+    addr = 0;
+    int max_count = _reset_vmp_ranges();
+    intptr_t * cursor = vmp_ranges;
+    cursor[0] = -1;
+
+    do {
+        // extract the top info using vm_region
+        count = VM_REGION_TOP_INFO_COUNT;
+        vmsize = 0;
+        kr = mach_vm_region(task, &addr, &vmsize, VM_REGION_TOP_INFO,
+                          (vm_region_info_t)&topinfo, &count, &obj);
+        if (kr == KERN_SUCCESS) {
+            vm_address_t start = addr, end = addr + vmsize;
+            // dladdr now gives the path of the shared object
+            Dl_info info;
+            if (dladdr((const void*)start, &info) == 0) {
+                // could not find image containing start
+                addr += vmsize;
+                continue;
+            }
+            if (_ignore_symbols_from_path(info.dli_fname)) {
+                // realloc if the chunk is to small
+                max_count = _resize_ranges(&cursor, max_count);
+                cursor = _add_to_range(cursor, start, end);
+            }
+            addr = addr + vmsize;
+        } else if (kr != KERN_INVALID_ADDRESS) {
+            goto teardown;
+        }
+    } while (kr == KERN_SUCCESS);
+
+    ret = 1;
+
+teardown:
+    if (task != MACH_PORT_NULL) {
+        mach_port_deallocate(mach_task_self(), task);
+    }
+    return ret;
+}
+#endif
+
+int vmp_native_enable(void) {
+    vmp_native_traces_enabled = 1;
+
+#if defined(__unix__)
+    return vmp_read_vmaps("/proc/self/maps");
+#elif defined(__APPLE__)
+    return vmp_read_vmaps(NULL);
+#endif
+}
+
+void vmp_native_disable(void) {
+    vmp_native_traces_enabled = 0;
+    if (vmp_ranges != NULL) {
+        free(vmp_ranges);
+        vmp_ranges = NULL;
+    }
+    vmp_range_count = 0;
+}
+
+int vmp_ignore_ip(intptr_t ip) {
+    int i = vmp_binary_search_ranges(ip, vmp_ranges, vmp_range_count);
+    if (i == -1) {
+        return 0;
+    }
+
+    assert((i & 1) == 0 && "returned index MUST be even");
+
+    intptr_t v = vmp_ranges[i];
+    intptr_t v2 = vmp_ranges[i+1];
+    return v <= ip && ip <= v2;
+}
+
+int vmp_binary_search_ranges(intptr_t ip, intptr_t * l, int count) {
+    intptr_t * r = l + count;
+    intptr_t * ol = l;
+    intptr_t * or = r-1;
+    while (1) {
+        ptrdiff_t i = (r-l)/2;
+        if (i == 0) {
+            if (l == ol && *l > ip) {
+                // at the start
+                return -1;
+            } else if (l == or && *l < ip) {
+                // at the end
+                return -1;
+            } else {
+                // we found the lower bound
+                i = l - ol;
+                if ((i & 1) == 1) {
+                    return i-1;
+                }
+                return i;
+            }
+        }
+        intptr_t * m = l + i;
+        if (ip < *m) {
+            r = m;
+        } else {
+            l = m;
+        }
+    }
+    return -1;
+}
+
+int vmp_ignore_symbol_count(void) {
+    return vmp_range_count;
+}
+
+intptr_t * vmp_ignore_symbols(void) {
+    return vmp_ranges;
+}
+
+void vmp_set_ignore_symbols(intptr_t * symbols, int count) {
+    vmp_ranges = symbols;
+    vmp_range_count = count;
+}
+#endif
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.h b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include "vmprof.h"
+
+int vmp_walk_and_record_stack(PY_STACK_FRAME_T * frame, void **data,
+                              int max_depth, int native_skip, intptr_t pc);
+
+int vmp_native_enabled(void);
+int vmp_native_enable(void);
+int vmp_ignore_ip(intptr_t ip);
+int vmp_binary_search_ranges(intptr_t ip, intptr_t * l, int count);
+int vmp_native_symbols_read(void);
+void vmp_profile_lines(int lines);
+int vmp_profiles_python_lines(void);
+
+int vmp_ignore_symbol_count(void);
+intptr_t * vmp_ignore_symbols(void);
+void vmp_set_ignore_symbols(intptr_t * symbols, int count);
+void vmp_native_disable(void);
+
+#ifdef __unix__
+int vmp_read_vmaps(const char * fname);
+#endif
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -43,6 +43,7 @@
 #define FRAME_CODE(f) f->
 PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag);
 #else
+#define RPY_EXTERN
 // for cpython
 #include "_vmprof.h"
 #include <Python.h>
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -141,6 +141,7 @@
 #  define _Py_atomic_load_relaxed(pp)  (*(pp))
 #endif
 
+#ifdef RPYTHON_VMPROF
 #ifndef RPYTHON_LL2CTYPES
 static PY_STACK_FRAME_T *get_vmprof_stack(void)
 {
@@ -158,7 +159,6 @@
 }
 #endif
 
-#ifdef RPYTHON_VMPROF
 RPY_EXTERN
 intptr_t vmprof_get_traceback(void *stack, void *ucontext,
                               intptr_t *result_p, intptr_t result_length)
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -92,12 +92,13 @@
 int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc)
 {
     PY_STACK_FRAME_T * frame;
+#ifdef RPYTHON_VMPROF
+    // do nothing here, 
+    current = (PY_STACK_FRAME_T*)frame;
+#else
     if (!current) {
         return 0;
     }
-#ifdef RPYTHON_VMPROF
-    frame = get_vmprof_stack();
-#else
     frame = current->frame;
 #endif
     // skip over
@@ -135,7 +136,7 @@
 #ifdef RPYTHON_VMPROF
     depth = get_stack_trace(get_vmprof_stack(), st->stack, MAX_STACK_DEPTH-1, (intptr_t)GetPC(uc));
 #else
-    depth = get_stack_trace(tstate, st->stack, MAX_STACK_DEPTH-1, NULL);
+    depth = get_stack_trace(tstate, st->stack, MAX_STACK_DEPTH-1, (intptr_t)NULL);
 #endif
     if (depth == 0) {
         return 0;
@@ -264,14 +265,18 @@
 static void atfork_disable_timer(void) {
     if (profile_interval_usec > 0) {
         remove_sigprof_timer();
+#ifndef RPYTHON_VMPROF
         is_enabled = 0;
+#endif
     }
 }
 
 static void atfork_enable_timer(void) {
     if (profile_interval_usec > 0) {
         install_sigprof_timer();
+#ifndef RPYTHON_VMPROF
         is_enabled = 1;
+#endif
     }
 }
 
@@ -302,9 +307,56 @@
     return 0;
 }
 
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+void init_cpyprof(int native)
+{
+    // skip this if native should not be enabled
+    if (!native) {
+        vmp_native_disable();
+        return;
+    }
+#if CPYTHON_HAS_FRAME_EVALUATION
+    PyThreadState *tstate = PyThreadState_GET();
+    tstate->interp->eval_frame = vmprof_eval;
+    _default_eval_loop = _PyEval_EvalFrameDefault;
+#elif defined(RPYTHON_VMPROF)
+    // TODO nothing?
+#else
+    if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx,
+                vmprof_eval, (void*)&_default_eval_loop) == 0) {
+    } else {
+        fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n");
+        // TODO dump the first few bytes and tell them to create an issue!
+        exit(-1);
+    }
+#endif
+    vmp_native_enable();
+}
+
+static void disable_cpyprof(void)
+{
+    vmp_native_disable();
+#if CPYTHON_HAS_FRAME_EVALUATION
+    PyThreadState *tstate = PyThreadState_GET();
+    tstate->interp->eval_frame = _PyEval_EvalFrameDefault;
+#elif defined(RPYTHON_VMPROF)
+    // TODO nothing?
+#else
+    if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) {
+        fprintf(stderr, "FATAL: could not remove trampoline\n");
+        exit(-1);
+    }
+#endif
+    dump_native_symbols(vmp_profile_fileno());
+}
+#endif
+
 RPY_EXTERN
-int vmprof_enable(int memory)
+int vmprof_enable(int memory, int native)
 {
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    init_cpyprof(native);
+#endif
     assert(vmp_profile_fileno() >= 0);
     assert(prepare_interval_usec > 0);
     profile_interval_usec = prepare_interval_usec;
@@ -326,7 +378,7 @@
 }
 
 
-static int close_profile(void)
+int close_profile(void)
 {
     (void)vmp_write_time_now(MARKER_TRAILER);
 
@@ -341,6 +393,9 @@
 {
     vmprof_ignore_signals(1);
     profile_interval_usec = 0;
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    disable_cpyprof();
+#endif
 
     if (remove_sigprof_timer() == -1)
         return -1;

From pypy.commits at gmail.com  Tue Feb  7 09:22:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 06:22:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix and clean up of f-strings: now
 internally handled as utf-8, instead of
Message-ID: <5899d808.9d711c0a.875ba.4554@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89999:4fd6ad6559bf
Date: 2017-02-07 13:39 +0100
http://bitbucket.org/pypy/pypy/changeset/4fd6ad6559bf/

Log:	fix and clean up of f-strings: now internally handled as utf-8,
	instead of as unicode-that-can-only-contain-ascii-anyway

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.pyparser import parsestring
 from pypy.interpreter import error
 from pypy.interpreter import unicodehelper
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rstring import StringBuilder
 
 
 def add_constant_string(astbuilder, joined_pieces, w_string, atom_node):
@@ -44,7 +44,8 @@
     if astbuilder.recursive_parser is None:
         astbuilder.error("internal error: parser not available for parsing "
                    "the expressions inside the f-string", atom_node)
-    source = '(%s)' % source.encode('utf-8')
+    assert isinstance(source, str)    # utf-8 encoded
+    source = '(%s)' % source
 
     info = pyparse.CompileInfo("<fstring>", "eval",
                                consts.PyCF_SOURCE_IS_UTF8 |
@@ -85,13 +86,13 @@
 
     # The first char must be a left brace, or we wouldn't have gotten
     # here. Skip over it.
-    u = fstr.unparsed
+    s = fstr.unparsed
     i = fstr.current_index
-    assert u[i] == u'{'
+    assert s[i] == '{'
     i += 1
 
     expr_start = i
-    while i < len(u):
+    while i < len(s):
 
         # Loop invariants.
         assert nested_depth >= 0
@@ -100,9 +101,9 @@
         else:
             assert string_type == 0
 
-        ch = u[i]
+        ch = s[i]
         # Nowhere inside an expression is a backslash allowed.
-        if ch == u'\\':
+        if ch == '\\':
             # Error: can't include a backslash character, inside
             # parens or strings or not.
             astbuilder.error("f-string expression part "
@@ -116,7 +117,7 @@
                 # Does this match the string_type (single or triple
                 # quoted)?
                 if string_type == 3:
-                    if i + 2 < len(u) and u[i + 1] == u[i + 2] == ch:
+                    if i + 2 < len(s) and s[i + 1] == s[i + 2] == ch:
                         # We're at the end of a triple quoted string.
                         i += 3
                         string_type = 0
@@ -128,9 +129,9 @@
                     string_type = 0
                     quote_char = 0
                     continue
-        elif ch == u"'" or ch == u'"':
+        elif ch == "'" or ch == '"':
             # Is this a triple quoted string?
-            if i + 2 < len(u) and u[i + 1] == u[i + 2] == ch:
+            if i + 2 < len(s) and s[i + 1] == s[i + 2] == ch:
                 string_type = 3
                 i += 2
             else:
@@ -138,20 +139,20 @@
                 string_type = 1
             # Start looking for the end of the string.
             quote_char = ord(ch)
-        elif ch in u"[{(":
+        elif ch in "[{(":
             nested_depth += 1
-        elif nested_depth != 0 and ch in u"]})":
+        elif nested_depth != 0 and ch in "]})":
             nested_depth -= 1
-        elif ch == u'#':
+        elif ch == '#':
             # Error: can't include a comment character, inside parens
             # or not.
             astbuilder.error("f-string expression part cannot include '#'",
                              atom_node)
-        elif nested_depth == 0 and ch in u"!:}":
+        elif nested_depth == 0 and ch in "!:}":
             # First, test for the special case of "!=". Since '=' is
             # not an allowed conversion character, nothing is lost in
             # this test.
-            if ch == '!' and i + 1 < len(u) and u[i+1] == u'=':
+            if ch == '!' and i + 1 < len(s) and s[i+1] == '=':
                 # This isn't a conversion character, just continue.
                 i += 1
                 continue
@@ -171,33 +172,33 @@
     if nested_depth:
         astbuilder.error("f-string: mismatched '(', '{' or '['", atom_node)
 
-    if i >= len(u):
+    if i >= len(s):
         unexpected_end_of_string(astbuilder, atom_node)
 
     # Compile the expression as soon as possible, so we show errors
     # related to the expression before errors related to the
     # conversion or format_spec.
-    expr = f_string_compile(astbuilder, u[expr_start:i], atom_node)
+    expr = f_string_compile(astbuilder, s[expr_start:i], atom_node)
     assert isinstance(expr, ast.Expression)
 
     # Check for a conversion char, if present.
-    if u[i] == u'!':
+    if s[i] == '!':
         i += 1
-        if i >= len(u):
+        if i >= len(s):
             unexpected_end_of_string(astbuilder, atom_node)
 
-        conversion = ord(u[i])
+        conversion = ord(s[i])
         i += 1
         if conversion not in (ord('s'), ord('r'), ord('a')):
             astbuilder.error("f-string: invalid conversion character: "
                              "expected 's', 'r', or 'a'", atom_node)
 
     # Check for the format spec, if present.
-    if i >= len(u):
+    if i >= len(s):
         unexpected_end_of_string(astbuilder, atom_node)
-    if u[i] == u':':
+    if s[i] == ':':
         i += 1
-        if i >= len(u):
+        if i >= len(s):
             unexpected_end_of_string(astbuilder, atom_node)
         fstr.current_index = i
         subpieces = []
@@ -205,7 +206,7 @@
         format_spec = f_string_to_ast_node(astbuilder, subpieces, atom_node)
         i = fstr.current_index
 
-    if i >= len(u) or u[i] != u'}':
+    if i >= len(s) or s[i] != '}':
         unexpected_end_of_string(astbuilder, atom_node)
 
     # We're at a right brace. Consume it.
@@ -222,7 +223,7 @@
 def fstring_find_literal(astbuilder, fstr, atom_node, rec):
     # Return the next literal part.  Updates the current index inside 'fstr'.
     # Differs from CPython: this version handles double-braces on its own.
-    u = fstr.unparsed
+    s = fstr.unparsed
     literal_start = fstr.current_index
     in_named_escape = False
 
@@ -230,21 +231,21 @@
     # brace (which isn't part of a unicode name escape such as
     # "\N{EULER CONSTANT}"), or the end of the string.
     i = literal_start
-    builder = UnicodeBuilder()
-    while i < len(u):
-        ch = u[i]
-        if (not in_named_escape and ch == u'{' and i - literal_start >= 2
-                and u[i - 2] == u'\\' and u[i - 1] == u'N'):
+    builder = StringBuilder()
+    while i < len(s):
+        ch = s[i]
+        if (not in_named_escape and ch == '{' and i - literal_start >= 2
+                and s[i - 2] == '\\' and s[i - 1] == 'N'):
             in_named_escape = True
-        elif in_named_escape and ch == u'}':
+        elif in_named_escape and ch == '}':
             in_named_escape = False
-        elif ch == u'{' or ch == u'}':
+        elif ch == '{' or ch == '}':
             # Check for doubled braces, but only at the top level. If
             # we checked at every level, then f'{0:{3}}' would fail
             # with the two closing braces.
-            if rec == 0 and i + 1 < len(u) and u[i + 1] == ch:
+            if rec == 0 and i + 1 < len(s) and s[i + 1] == ch:
                 i += 1   # skip over the second brace
-            elif rec == 0 and ch == u'}':
+            elif rec == 0 and ch == '}':
                 # Where a single '{' is the start of a new expression, a
                 # single '}' is not allowed.
                 astbuilder.error("f-string: single '}' is not allowed",
@@ -259,14 +260,13 @@
 
     fstr.current_index = i
     literal = builder.build()
-    if not fstr.raw_mode and u'\\' in literal:
-        # xxx messy
+    if not fstr.raw_mode and '\\' in literal:
         space = astbuilder.space
-        literal = literal.encode('utf-8')
         literal = parsestring.decode_unicode_utf8(space, literal, 0,
                                                   len(literal))
-        literal = unicodehelper.decode_unicode_escape(space, literal)
-    return literal
+        return unicodehelper.decode_unicode_escape(space, literal)
+    else:
+        return literal.decode('utf-8')
 
 
 def fstring_find_literal_and_expr(astbuilder, fstr, atom_node, rec):
@@ -274,15 +274,15 @@
     # following expression node.  Updates the current index inside 'fstr'.
     literal = fstring_find_literal(astbuilder, fstr, atom_node, rec)
 
-    u = fstr.unparsed
+    s = fstr.unparsed
     i = fstr.current_index
-    if i >= len(u) or u[i] == u'}':
+    if i >= len(s) or s[i] == '}':
         # We're at the end of the string or the end of a nested
         # f-string: no expression.
         expr = None
     else:
         # We must now be the start of an expression, on a '{'.
-        assert u[i] == u'{'
+        assert s[i] == '{'
         expr = fstring_find_expr(astbuilder, fstr, atom_node, rec)
     return literal, expr
 
@@ -320,14 +320,16 @@
         astbuilder.error("f-string: unexpected end of string", atom_node)
 
     if rec != 0 and (fstr.current_index >= len(fstr.unparsed) or
-                     fstr.unparsed[fstr.current_index] != u'}'):
+                     fstr.unparsed[fstr.current_index] != '}'):
         astbuilder.error("f-string: expecting '}'", atom_node)
 
 
 def f_string_to_ast_node(astbuilder, joined_pieces, atom_node):
     # remove empty Strs
+    space = astbuilder.space
     values = [node for node in joined_pieces
-                   if not (isinstance(node, ast.Str) and not node.s)]
+                   if not isinstance(node, ast.Str)
+                      or space.is_true(node.s)]
     if len(values) > 1:
         return ast.JoinedStr(values, atom_node.get_lineno(),
                                      atom_node.get_column())
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1194,6 +1194,38 @@
         raises(SyntaxError, self.run, "f'''{5)\n#}'''")
         raises(SyntaxError, self.run, "f'\\x'")
 
+    def test_fstring_encoding(self):
+        src = """# -*- coding: latin-1 -*-\nz=ord(f'{"\xd8"}')\n"""
+        yield self.st, src, 'z', 0xd8
+        src = """# -*- coding: utf-8 -*-\nz=ord(f'{"\xc3\x98"}')\n"""
+        yield self.st, src, 'z', 0xd8
+
+        src = """z=ord(f'\\xd8')"""
+        yield self.st, src, 'z', 0xd8
+        src = """z=ord(f'\\u00d8')"""
+        yield self.st, src, 'z', 0xd8
+
+        src = """# -*- coding: latin-1 -*-\nz=ord(f'\xd8')\n"""
+        yield self.st, src, 'z', 0xd8
+        src = """# -*- coding: utf-8 -*-\nz=ord(f'\xc3\x98')\n"""
+        yield self.st, src, 'z', 0xd8
+
+    def test_fstring_encoding_r(self):
+        src = """# -*- coding: latin-1 -*-\nz=ord(fr'{"\xd8"}')\n"""
+        yield self.st, src, 'z', 0xd8
+        src = """# -*- coding: utf-8 -*-\nz=ord(rf'{"\xc3\x98"}')\n"""
+        yield self.st, src, 'z', 0xd8
+
+        src = """z=fr'\\xd8'"""
+        yield self.st, src, 'z', "\\xd8"
+        src = """z=rf'\\u00d8'"""
+        yield self.st, src, 'z', "\\u00d8"
+
+        src = """# -*- coding: latin-1 -*-\nz=ord(rf'\xd8')\n"""
+        yield self.st, src, 'z', 0xd8
+        src = """# -*- coding: utf-8 -*-\nz=ord(fr'\xc3\x98')\n"""
+        yield self.st, src, 'z', 0xd8
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -7,7 +7,7 @@
 
 class W_FString(W_Root):
     def __init__(self, unparsed, raw_mode):
-        assert isinstance(unparsed, unicode)
+        assert isinstance(unparsed, str)    # utf-8 encoded string
         self.unparsed = unparsed     # but the quotes are removed
         self.raw_mode = raw_mode
         self.current_index = 0       # for astcompiler.fstring
@@ -85,14 +85,13 @@
         q -= 2
 
     if unicode_literal and not rawmode: # XXX Py_UnicodeFlag is ignored for now
+        assert 0 <= ps <= q
+        if saw_f:
+            return W_FString(s[ps:q], rawmode)
         if encoding is None:
-            assert 0 <= ps <= q
             substr = s[ps:q]
         else:
             substr = decode_unicode_utf8(space, s, ps, q)
-        if saw_f:
-            v = unicodehelper.decode_utf8(space, substr)
-            return W_FString(v, rawmode)
         v = unicodehelper.decode_unicode_escape(space, substr)
         return space.wrap(v)
 
@@ -109,10 +108,10 @@
     if rawmode or '\\' not in substr:
         if not unicode_literal:
             return space.newbytes(substr)
+        elif saw_f:
+            return W_FString(substr, rawmode)
         else:
             v = unicodehelper.decode_utf8(space, substr)
-            if saw_f:
-                return W_FString(v, rawmode)
             return space.wrap(v)
 
     v = PyString_DecodeEscape(space, substr, 'strict', encoding)

From pypy.commits at gmail.com  Tue Feb  7 09:48:27 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 07 Feb 2017 06:48:27 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: add brackets to if statement,
 swap variable names
Message-ID: <5899de3b.2687df0a.b4242.a5ec@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90000:c7f8f45b5a55
Date: 2017-02-07 15:47 +0100
http://bitbucket.org/pypy/pypy/changeset/c7f8f45b5a55/

Log:	add brackets to if statement, swap variable names

diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -169,8 +169,9 @@
 #else
     intptr_t pc = ucontext ? (intptr_t)GetPC((ucontext_t *)ucontext) : 0;
 #endif
-    if (stack == NULL)
+    if (stack == NULL) {
         stack = get_vmprof_stack();
+    }
     n = get_stack_trace(stack, result_p, result_length - 2, pc);
     return (intptr_t)n;
 }
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -94,7 +94,7 @@
     PY_STACK_FRAME_T * frame;
 #ifdef RPYTHON_VMPROF
     // do nothing here, 
-    current = (PY_STACK_FRAME_T*)frame;
+    frame = (PY_STACK_FRAME_T*)current;
 #else
     if (!current) {
         return 0;

From pypy.commits at gmail.com  Tue Feb  7 10:28:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 07:28:32 -0800 (PST)
Subject: [pypy-commit] cffi default: ffi.addressof(lib,
 "name") now also works in in-line mode
Message-ID: <5899e7a0.5190df0a.265de.abbf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2886:4174b2bf46e0
Date: 2017-02-07 16:28 +0100
http://bitbucket.org/cffi/cffi/changeset/4174b2bf46e0/

Log:	ffi.addressof(lib, "name") now also works in in-line mode

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3691,19 +3691,14 @@
     CTypeDescrObject *ct;
     char *funcname;
     void *funcptr;
-    int ok;
 
     if (!PyArg_ParseTuple(args, "O!s:load_function",
                           &CTypeDescr_Type, &ct, &funcname))
         return NULL;
 
-    ok = 0;
-    if (ct->ct_flags & CT_FUNCTIONPTR)
-        ok = 1;
-    if ((ct->ct_flags & CT_POINTER) && (ct->ct_itemdescr->ct_flags & CT_VOID))
-        ok = 1;
-    if (!ok) {
-        PyErr_Format(PyExc_TypeError, "function cdata expected, got '%s'",
+    if (!(ct->ct_flags & (CT_FUNCTIONPTR | CT_POINTER | CT_ARRAY))) {
+        PyErr_Format(PyExc_TypeError,
+                     "function or pointer or array cdata expected, got '%s'",
                      ct->ct_name);
         return NULL;
     }
@@ -3711,12 +3706,15 @@
     funcptr = dlsym(dlobj->dl_handle, funcname);
     if (funcptr == NULL) {
         const char *error = dlerror();
-        PyErr_Format(PyExc_KeyError,
-                     "function '%s' not found in library '%s': %s",
+        PyErr_Format(PyExc_AttributeError,
+                     "function/symbol '%s' not found in library '%s': %s",
                      funcname, dlobj->dl_name, error);
         return NULL;
     }
 
+    if ((ct->ct_flags & CT_ARRAY) && ct->ct_length < 0) {
+        ct = (CTypeDescrObject *)ct->ct_stuff;
+    }
     return new_simple_cdata(funcptr, ct);
 }
 
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -462,7 +462,12 @@
         field or array item in the structure or array, recursively in
         case of nested structures.
         """
-        ctype = self._backend.typeof(cdata)
+        try:
+            ctype = self._backend.typeof(cdata)
+        except TypeError:
+            if '__addressof__' in type(cdata).__dict__:
+                return type(cdata).__addressof__(cdata, *fields_or_indexes)
+            raise
         if fields_or_indexes:
             ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
         else:
@@ -775,10 +780,7 @@
         key = 'function ' + name
         tp, _ = ffi._parser._declarations[key]
         BType = ffi._get_cached_btype(tp)
-        try:
-            value = backendlib.load_function(BType, name)
-        except KeyError as e:
-            raise AttributeError('%s: %s' % (name, e))
+        value = backendlib.load_function(BType, name)
         library.__dict__[name] = value
     #
     def accessor_variable(name):
@@ -791,6 +793,21 @@
             lambda self: read_variable(BType, name),
             lambda self, value: write_variable(BType, name, value)))
     #
+    def addressof_var(name):
+        try:
+            return addr_variables[name]
+        except KeyError:
+            with ffi._lock:
+                if name not in addr_variables:
+                    key = 'variable ' + name
+                    tp, _ = ffi._parser._declarations[key]
+                    BType = ffi._get_cached_btype(tp)
+                    if BType.kind != 'array':
+                        BType = model.pointer_cache(ffi, BType)
+                    p = backendlib.load_function(BType, name)
+                    addr_variables[name] = p
+            return addr_variables[name]
+    #
     def accessor_constant(name):
         raise NotImplementedError("non-integer constant '%s' cannot be "
                                   "accessed from a dlopen() library" % (name,))
@@ -800,6 +817,7 @@
     #
     accessors = {}
     accessors_version = [False]
+    addr_variables = {}
     #
     def update_accessors():
         if accessors_version[0] is ffi._cdef_version:
@@ -850,6 +868,18 @@
             with ffi._lock:
                 update_accessors()
                 return accessors.keys()
+        def __addressof__(self, name):
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            make_accessor(name)
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            raise AttributeError("cffi library has no function or "
+                                 "global variable named '%s'" % (name,))
     #
     if libname is not None:
         try:
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -46,6 +46,10 @@
 * ``ffi.buffer`` is now the name of cffi's buffer type, and
   ``ffi.buffer()`` works like before but is the constructor of that type.
 
+* ``ffi.addressof(lib, "name")``  now works also in in-line mode, not
+  only in out-of-line mode.  This is useful for taking the address of
+  global variables.
+
 
 v1.9
 ====
diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py
--- a/testing/cffi0/test_ownlib.py
+++ b/testing/cffi0/test_ownlib.py
@@ -282,3 +282,21 @@
             assert ret.right == ownlib.right
             assert ret.top == ownlib.top
             assert ret.bottom == ownlib.bottom
+
+    def test_addressof_lib(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("long left; int test_getting_errno(void);")
+        lib = ffi.dlopen(self.module)
+        lib.left = 123456
+        p = ffi.addressof(lib, "left")
+        assert ffi.typeof(p) == ffi.typeof("long *")
+        assert p[0] == 123456
+        p[0] += 1
+        assert lib.left == 123457
+        pfn = ffi.addressof(lib, "test_getting_errno")
+        assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
+        assert pfn == lib.test_getting_errno

From pypy.commits at gmail.com  Tue Feb  7 10:31:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 07:31:56 -0800 (PST)
Subject: [pypy-commit] cffi default: fix test
Message-ID: <5899e86c.068f1c0a.ef4bf.5b98@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2887:6f5001375739
Date: 2017-02-07 16:31 +0100
http://bitbucket.org/cffi/cffi/changeset/6f5001375739/

Log:	fix test

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -376,7 +376,7 @@
     x = find_and_load_library(None)
     BVoidP = new_pointer_type(new_void_type())
     assert x.load_function(BVoidP, 'strcpy')
-    py.test.raises(KeyError, x.load_function,
+    py.test.raises(AttributeError, x.load_function,
                    BVoidP, 'xxx_this_function_does_not_exist')
     # the next one is from 'libm', not 'libc', but we assume
     # that it is already loaded too, so it should work

From pypy.commits at gmail.com  Tue Feb  7 10:37:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 07:37:11 -0800 (PST)
Subject: [pypy-commit] pypy default: update to cffi/6f5001375739
Message-ID: <5899e9a7.9da0df0a.ff6a7.afed@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90001:f63a701a6ae4
Date: 2017-02-07 16:36 +0100
http://bitbucket.org/pypy/pypy/changeset/f63a701a6ae4/

Log:	update to cffi/6f5001375739

diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -40,25 +40,22 @@
 
     @unwrap_spec(w_ctype=W_CType, name=str)
     def load_function(self, w_ctype, name):
-        from pypy.module._cffi_backend import ctypefunc, ctypeptr, ctypevoid
+        from pypy.module._cffi_backend import ctypeptr, ctypearray
         space = self.space
         #
-        ok = False
-        if isinstance(w_ctype, ctypefunc.W_CTypeFunc):
-            ok = True
-        if (isinstance(w_ctype, ctypeptr.W_CTypePointer) and
-            isinstance(w_ctype.ctitem, ctypevoid.W_CTypeVoid)):
-            ok = True
-        if not ok:
+        if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray):
             raise oefmt(space.w_TypeError,
-                        "function cdata expected, got '%s'", w_ctype.name)
+                        "function or pointer or array cdata expected, got '%s'",
+                        w_ctype.name)
         #
         try:
             cdata = dlsym(self.handle, name)
         except KeyError:
-            raise oefmt(space.w_KeyError,
-                        "function '%s' not found in library '%s'",
+            raise oefmt(space.w_AttributeError,
+                        "function/symbol '%s' not found in library '%s'",
                         name, self.name)
+        if isinstance(w_ctype, ctypearray.W_CTypeArray) and w_ctype.length < 0:
+            w_ctype = w_ctype.ctptr
         return W_CData(space, rffi.cast(rffi.CCHARP, cdata), w_ctype)
 
     @unwrap_spec(w_ctype=W_CType, name=str)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -365,7 +365,7 @@
     x = find_and_load_library(None)
     BVoidP = new_pointer_type(new_void_type())
     assert x.load_function(BVoidP, 'strcpy')
-    py.test.raises(KeyError, x.load_function,
+    py.test.raises(AttributeError, x.load_function,
                    BVoidP, 'xxx_this_function_does_not_exist')
     # the next one is from 'libm', not 'libc', but we assume
     # that it is already loaded too, so it should work

From pypy.commits at gmail.com  Tue Feb  7 10:37:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Feb 2017 07:37:48 -0800 (PST)
Subject: [pypy-commit] pypy default: update to cffi/6f5001375739 (forgot
 this)
Message-ID: <5899e9cc.92d41c0a.8f40b.6011@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90002:c645557d7175
Date: 2017-02-07 16:37 +0100
http://bitbucket.org/pypy/pypy/changeset/c645557d7175/

Log:	update to cffi/6f5001375739 (forgot this)

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -462,7 +462,12 @@
         field or array item in the structure or array, recursively in
         case of nested structures.
         """
-        ctype = self._backend.typeof(cdata)
+        try:
+            ctype = self._backend.typeof(cdata)
+        except TypeError:
+            if '__addressof__' in type(cdata).__dict__:
+                return type(cdata).__addressof__(cdata, *fields_or_indexes)
+            raise
         if fields_or_indexes:
             ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
         else:
@@ -775,10 +780,7 @@
         key = 'function ' + name
         tp, _ = ffi._parser._declarations[key]
         BType = ffi._get_cached_btype(tp)
-        try:
-            value = backendlib.load_function(BType, name)
-        except KeyError as e:
-            raise AttributeError('%s: %s' % (name, e))
+        value = backendlib.load_function(BType, name)
         library.__dict__[name] = value
     #
     def accessor_variable(name):
@@ -791,6 +793,21 @@
             lambda self: read_variable(BType, name),
             lambda self, value: write_variable(BType, name, value)))
     #
+    def addressof_var(name):
+        try:
+            return addr_variables[name]
+        except KeyError:
+            with ffi._lock:
+                if name not in addr_variables:
+                    key = 'variable ' + name
+                    tp, _ = ffi._parser._declarations[key]
+                    BType = ffi._get_cached_btype(tp)
+                    if BType.kind != 'array':
+                        BType = model.pointer_cache(ffi, BType)
+                    p = backendlib.load_function(BType, name)
+                    addr_variables[name] = p
+            return addr_variables[name]
+    #
     def accessor_constant(name):
         raise NotImplementedError("non-integer constant '%s' cannot be "
                                   "accessed from a dlopen() library" % (name,))
@@ -800,6 +817,7 @@
     #
     accessors = {}
     accessors_version = [False]
+    addr_variables = {}
     #
     def update_accessors():
         if accessors_version[0] is ffi._cdef_version:
@@ -850,6 +868,18 @@
             with ffi._lock:
                 update_accessors()
                 return accessors.keys()
+        def __addressof__(self, name):
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            make_accessor(name)
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            raise AttributeError("cffi library has no function or "
+                                 "global variable named '%s'" % (name,))
     #
     if libname is not None:
         try:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1227,6 +1227,26 @@
         assert list(a)[:1000] + [0] * (len(a)-1000) == list(b)
         f.close()
 
+    def test_ffi_buffer_comparisons(self):
+        ffi = FFI(backend=self.Backend())
+        ba = bytearray(range(100, 110))
+        assert ba == memoryview(ba)    # justification for the following
+        a = ffi.new("uint8_t[]", list(ba))
+        c = ffi.new("uint8_t[]", [99] + list(ba))
+        try:
+            b_full = ffi.buffer(a)
+            b_short = ffi.buffer(a, 3)
+            b_mid = ffi.buffer(a, 6)
+            b_other = ffi.buffer(c, 6)
+        except NotImplementedError as e:
+            py.test.skip(str(e))
+        else:
+            content = b_full[:]
+            assert content == b_full == ba
+            assert b_other < b_short < b_mid < b_full
+            assert ba > b_mid > ba[0:2]
+            assert b_short != ba[1:4]
+
     def test_array_in_struct(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct foo_s { int len; short data[5]; };")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -193,9 +193,12 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
+        buff1 = ffi.buffer(s)
         t = lib.try_with_value(fnames.index(name), value)
         raw2 = ffi.buffer(t, len(raw1))[:]
         assert raw1 == raw2
+        buff2 = ffi.buffer(t, len(buff1))
+        assert buff1 == buff2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -283,3 +283,21 @@
             assert ret.right == ownlib.right
             assert ret.top == ownlib.top
             assert ret.bottom == ownlib.bottom
+
+    def test_addressof_lib(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("long left; int test_getting_errno(void);")
+        lib = ffi.dlopen(self.module)
+        lib.left = 123456
+        p = ffi.addressof(lib, "left")
+        assert ffi.typeof(p) == ffi.typeof("long *")
+        assert p[0] == 123456
+        p[0] += 1
+        assert lib.left == 123457
+        pfn = ffi.addressof(lib, "test_getting_errno")
+        assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
+        assert pfn == lib.test_getting_errno

From pypy.commits at gmail.com  Tue Feb  7 12:58:07 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 07 Feb 2017 09:58:07 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: force function name
 __vmprof_eval_vmprof for the "trampoline", add native test
Message-ID: <589a0aaf.810b1c0a.dabbb.99b7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90003:389b5116a6a3
Date: 2017-02-07 18:57 +0100
http://bitbucket.org/pypy/pypy/changeset/389b5116a6a3/

Log:	force function name __vmprof_eval_vmprof for the "trampoline", add
	native test

diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -32,8 +32,8 @@
         return code._vmprof_unique_id
     return 0
 
-def enable(fileno, interval):
-    _get_vmprof().enable(fileno, interval)
+def enable(fileno, interval, memory=0, native=0):
+    _get_vmprof().enable(fileno, interval, memory, native)
 
 def disable():
     _get_vmprof().disable()
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -214,6 +214,7 @@
                 return decorated_jitted_function(unique_id, *args)
 
         decorated_function.__name__ = func.__name__ + '_rvmprof'
+        decorated_function.c_name = '__vmprof_eval_vmprof'
         return decorated_function
 
     return decorate
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -16,36 +16,6 @@
 #include "shared/vmprof_main_win32.h"
 #endif
 
-#ifdef VMPROF_UNIX
-#ifdef __clang__
-__attribute__((disable_tail_calls))
-#elif defined(__GNUC__)
-__attribute__((optimize("O1")))
-#endif
-PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag)
-{
-#ifdef X86_64
-    register PY_STACK_FRAME_T * callee_saved asm("rbx");
-#elif defined(X86_32)
-    register PY_STACK_FRAME_T * callee_saved asm("edi");
-#else
-#    error "platform not supported"
-#endif
-
-    asm volatile(
-#ifdef X86_64
-        "movq %1, %0\t\n"
-#elif defined(X86_32)
-        "mov %1, %0\t\n"
-#else
-#    error "platform not supported"
-#endif
-        : "=r" (callee_saved)
-        : "r" (f) );
-    return NULL; // TODO _default_eval_loop(f, throwflag);
-}
-#endif
-
 void dump_native_symbols(int fileno)
 {
 // TODO    PyObject * mod = NULL;
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -14,6 +14,7 @@
 static volatile int is_enabled = 0;
 static destructor Original_code_dealloc = 0;
 static PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
+void dump_native_symbols(int fileno);
 
 #if VMPROF_UNIX
 #include "trampoline.h"
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -195,7 +195,7 @@
         //}
 
 
-        if ((void*)pip.start_ip == (void*)vmprof_eval) {
+        if ((void*)pip.start_ip == (void*)VMPROF_EVAL()) {
             // yes we found one stack entry of the python frames!
             unw_word_t rbx = 0;
             if (unw_get_reg(&cursor, REG_RBX, &rbx) < 0) {
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -41,7 +41,9 @@
 #define PY_THREAD_STATE_T void
 #define FRAME_STEP(f) f->next
 #define FRAME_CODE(f) f->
-PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag);
+typedef long Signed;
+RPY_EXTERN Signed __vmprof_eval_vmprof();
+#define VMPROF_EVAL() __vmprof_eval_vmprof
 #else
 #define RPY_EXTERN
 // for cpython
@@ -54,6 +56,7 @@
 #define FRAME_STEP(f) f->f_back
 #define FRAME_CODE(f) f->f_code
 PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag);
+#define VMPROF_EVAL() vmprof_eval
 #endif
 
 
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -147,3 +147,78 @@
     finally:
         assert os.path.exists(tmpfilename)
         os.unlink(tmpfilename)
+
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rtyper.lltypesystem import rffi, lltype
+def test_native():
+    eci = ExternalCompilationInfo(compile_extra=['-g','-O2'],
+            separate_module_sources=["""
+            RPY_EXTERN int native_func(void) {
+                return 42;
+            }
+            """])
+
+    native_func = rffi.llexternal("native_func", [], rffi.INT,
+                                  compilation_info=eci)
+
+    class MyCode:
+        pass
+    def get_name(code):
+        return 'py:code:52:x'
+
+    try:
+        rvmprof.register_code_object_class(MyCode, get_name)
+    except rvmprof.VMProfPlatformUnsupported as e:
+        py.test.skip(str(e))
+
+    @rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
+    def main(code, num):
+        if num > 0:
+            return main(code, num-1)
+        else:
+            for i in range(100):
+                native_func()
+            return native_func()
+
+    tmpfilename = str(udir.join('test_rvmprof'))
+
+    def f():
+        if NonConstant(False):
+            # Hack to give os.open() the correct annotation
+            os.open('foo', 1, 1)
+        code = MyCode()
+        rvmprof.register_code(code, get_name)
+        fd = os.open(tmpfilename, os.O_WRONLY | os.O_CREAT, 0666)
+        if we_are_translated():
+            num = 100000000
+            period = 0.0001
+        else:
+            num = 10000
+            period = 0.9
+        rvmprof.enable(fd, period, native=1)
+        res = main(code, num)
+        #assert res == 499999500000
+        rvmprof.disable()
+        os.close(fd)
+        return 0
+
+    def check_profile(filename):
+        from vmprof import read_profile
+
+        prof = read_profile(filename)
+        assert prof.get_tree().name.startswith("py:")
+        assert prof.get_tree().count
+
+    #assert f() == 0
+    #assert os.path.exists(tmpfilename)
+    fn = compile(f, [], gcpolicy="minimark")
+    assert fn() == 0
+    try:
+        import vmprof
+    except ImportError:
+        py.test.skip("vmprof unimportable")
+    else:
+        check_profile(tmpfilename)
+    finally:
+        assert os.path.exists(tmpfilename)
+        os.unlink(tmpfilename)

From pypy.commits at gmail.com  Tue Feb  7 14:21:02 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 07 Feb 2017 11:21:02 -0800 (PST)
Subject: [pypy-commit] pypy default: Reimplementing every extension under
 the sun in cffi is not really that useful any more,
 recommend working with upstreams instead
Message-ID: <589a1e1e.810b1c0a.e6d0e.130e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90004:8a778fab7e8a
Date: 2017-02-07 19:20 +0000
http://bitbucket.org/pypy/pypy/changeset/8a778fab7e8a/

Log:	Reimplementing every extension under the sun in cffi is not really
	that useful any more, recommend working with upstreams instead

diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -176,28 +176,33 @@
 * `hg`
 
 
-Optimising cpyext (CPython C-API compatibility layer)
------------------------------------------------------
-
-A lot of work has gone into PyPy's implementation of CPython's C-API over
-the last years to let it reach a practical level of compatibility, so that
-C extensions for CPython work on PyPy without major rewrites. However,
-there are still many edges and corner cases where it misbehaves.
-
-The objective of this project is to fix bugs in cpyext and to optimise
-several performance critical parts of it, such as the reference counting
-support and other heavily used C-API functions. The net result would be to
-have CPython extensions run much faster on PyPy than they currently do, or
-to make them work at all if they currently don't. A part of this work would
-be to get cpyext into a shape where it supports running Cython generated
-extensions.
-
 ======================================
 Make more python modules pypy-friendly
 ======================================
 
-Work has been started on a few popular python packages. Here is a partial
-list of good work that needs to be finished:
+A lot of work has gone into PyPy's implementation of CPython's C-API, cpyext,
+over the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However, there
+are still many edges and corner cases where it misbehaves.
+
+For any popular extension that does not already advertise full PyPy
+compatibility, it would thus be useful to take a close look at it in order to
+make it fully compatible with PyPy. The general process is something like:
+
+* Run the extension's tests on PyPy and look at the test failures.
+* Some of the failures may be solved by identifying cases where the extension
+  relies on undocumented or internal details of CPython, and rewriting the
+  relevant code to follow documented best practices. Open issues and send pull
+  requests as appropriate given the extension's development process.
+* Other failures may highlight incompatibilities between cpyext and CPython.
+  Please report them to us and try to fix them.
+* Run benchmarks, either provided by the extension developers or created by
+  you. Any case where PyPy is significantly slower than CPython is to be
+  considered a bug and solved as above.
+
+Alternatively, an approach we used to recommend was to rewrite C extensions
+using more pypy-friendly technologies, e.g. cffi. Here is a partial list of
+good work that needs to be finished:
 
 **matplotlib** https://github.com/mattip/matplotlib
 
@@ -223,5 +228,3 @@
 **pyopengl** https://bitbucket.org/duangle/pyopengl-cffi
 
     Status: unknown
-
-

From pypy.commits at gmail.com  Tue Feb  7 16:35:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 07 Feb 2017 13:35:09 -0800 (PST)
Subject: [pypy-commit] pypy default: Add project idea: Explicit typing in
 RPython
Message-ID: <589a3d8d.ad8ddf0a.8cbd0.4e06@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90005:5b678fd8a673
Date: 2017-02-07 21:34 +0000
http://bitbucket.org/pypy/pypy/changeset/5b678fd8a673/

Log:	Add project idea: Explicit typing in RPython

diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -38,6 +38,21 @@
 .. _mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 
 
+Explicit typing in RPython
+--------------------------
+
+RPython is mostly based around type inference, but there are many cases where
+specifying types explicitly is useful. We would like to be able to optionally 
+specify the exact types of the arguments to any function. We already have
+solutions in that space, ``@rpython.rlib.objectmodel.enforceargs`` and
+``@rpython.rlib.signature.signature``, but they are inconvenient and limited. 
+For instance, they do not easily allow to express the type "dict with ints as
+keys and lists of instances of Foo as values".
+
+Additionally, we would like to be able to specify the types of instance
+attributes. Unlike the function case, this is likely to require some
+refactoring of the annotator. 
+
 Make bytearray type fast
 ------------------------
 

From pypy.commits at gmail.com  Wed Feb  8 05:13:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Feb 2017 02:13:28 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: fix merge
Message-ID: <589aef48.4b371c0a.df433.c019@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90007:e0be031fa6c2
Date: 2017-02-08 11:12 +0100
http://bitbucket.org/pypy/pypy/changeset/e0be031fa6c2/

Log:	fix merge

diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -4,7 +4,7 @@
 from rpython.annotator.model import UnionError
 from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote,
     JitHintError, oopspec, isconstant, conditional_call,
-    elidable, unroll_safe, dont_look_inside, conditional_call_edliable,
+    elidable, unroll_safe, dont_look_inside, conditional_call_elidable,
     enter_portal_frame, leave_portal_frame, elidable_compatible,
     RandomWeAreJittedTestMixin)
 from rpython.rlib.rarithmetic import r_uint

From pypy.commits at gmail.com  Wed Feb  8 05:13:30 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Feb 2017 02:13:30 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: avoid interaction with other
 usages of random.random()
Message-ID: <589aef4a.da6d1c0a.328b2.bcfc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90008:5d42d925abed
Date: 2017-02-08 11:12 +0100
http://bitbucket.org/pypy/pypy/changeset/5d42d925abed/

Log:	avoid interaction with other usages of random.random()

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -397,11 +397,12 @@
     def setup_method(self, meth):
         global _we_are_jitted_interpreted
         seed = random.random()
-        print "seed", seed
-        random.seed(seed)
+        print "RandomWeAreJittedTestMixin: seed", seed
+        r = random.Random()
+        r.seed(seed)
         self.orig_we_are_jitted = _we_are_jitted_interpreted
         def _we_are_jitted_interpreted_random():
-            result = random.random() > 0.5
+            result = r.random() > 0.5
             return result
         _we_are_jitted_interpreted = _we_are_jitted_interpreted_random
 

From pypy.commits at gmail.com  Wed Feb  8 08:13:06 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 08 Feb 2017 05:13:06 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges: merge default
Message-ID: <589b1962.46bb1c0a.7ccb4.0745@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges
Changeset: r90009:189d2e2db7ad
Date: 2017-02-08 11:39 +0100
http://bitbucket.org/pypy/pypy/changeset/189d2e2db7ad/

Log:	merge default

diff too long, truncating to 2000 out of 156212 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -77,3 +77,5 @@
 ^.hypothesis/
 ^release/
 ^rpython/_cache$
+
+pypy/module/cppyy/.+/*\.pcm
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,423 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation, BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-

From pypy.commits at gmail.com  Wed Feb  8 13:37:42 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 08 Feb 2017 10:37:42 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: some more changes to the native
 test
Message-ID: <589b6576.812d1c0a.edcce.8f6c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90010:d671b63850ea
Date: 2017-02-08 19:37 +0100
http://bitbucket.org/pypy/pypy/changeset/d671b63850ea/

Log:	some more changes to the native test

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -46,7 +46,7 @@
 
 eci_kwds = dict(
     include_dirs = [SRC, SHARED, BACKTRACE, UDIS86],
-    includes = ['rvmprof.h', 'vmprof_stack.h'],
+    includes = ['rvmprof.h','vmprof_stack.h'],
     libraries = _libs,
     separate_module_files = [
         SRC.join('rvmprof.c'),
@@ -99,6 +99,7 @@
     return CInterface(locals())
 
 
+
 class CInterface(object):
     def __init__(self, namespace):
         for k, v in namespace.iteritems():
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -16,16 +16,3 @@
 #include "shared/vmprof_main_win32.h"
 #endif
 
-void dump_native_symbols(int fileno)
-{
-// TODO    PyObject * mod = NULL;
-// TODO
-// TODO    mod = PyImport_ImportModuleNoBlock("vmprof");
-// TODO    if (mod == NULL)
-// TODO        goto error;
-// TODO
-// TODO    PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
-// TODO
-// TODOerror:
-// TODO    Py_XDECREF(mod);
-}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -21,7 +21,6 @@
 #define RPY_EXPORTED  extern __attribute__((visibility("default")))
 #endif
 
-
 RPY_EXTERN char *vmprof_init(int fd, double interval, int memory,
                      int lines, const char *interp_name, int native);
 RPY_EXTERN void vmprof_ignore_signals(int);
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -170,11 +170,6 @@
     }
 }
 
-#else
-// other platforms than linux & mac os x
-void dump_all_known_symbols(int fd) {
-    // oh, nothing to do!! a not supported platform
-}
 #endif
 
 #ifdef __unix__
@@ -247,3 +242,193 @@
 #endif
     return 0;
 }
+
+#ifdef RPYTHON_VMPROF
+
+#define WORD_SIZE sizeof(long)
+#define ADDR_SIZE sizeof(void*)
+#define MAXLEN 1024
+
+void _dump_native_symbol(int fileno, void * addr, char * sym, int linenumber, char * filename) {
+    char natsym[64];
+    off_t pos_before;
+    struct str {
+        void * addr;
+        // NOTE windows 64, not supported yet
+        long size;
+        char str[1024];
+    } s;
+    pos_before = lseek(fileno, 0, SEEK_CUR);
+    lseek(fileno, 0, SEEK_END);
+
+    s.addr = addr;
+    /* must mach '<lang>:<name>:<line>:<file>'
+     * 'n' has been chosen as lang here, because the symbol
+     * can be generated from several languages (e.g. C, C++, ...)
+     */
+    // MARKER_NATIVE_SYMBOLS is \x08
+    write(fileno, "\x08", 1);
+    if (sym == NULL || sym[0] == '\x00') {
+        snprintf(natsym, 64, "<native symbol %p>", addr);
+        sym = natsym;
+    }
+    if (filename != NULL) {
+        s.size = snprintf(s.str, 1024, "n:%s:%d:%s", sym, linenumber, filename);
+    } else {
+        s.size = snprintf(s.str, 1024, "n:%s:%d:-", sym, linenumber);
+    }
+    write(fileno, &s, sizeof(void*)+sizeof(long)+s.size);
+
+    lseek(fileno, pos_before, SEEK_SET);
+}
+
+int _skip_string(int fileno)
+{
+    long chars;
+    int count = read(fileno, &chars, sizeof(long));
+    LOG("reading string of %d chars\n", chars);
+    if (count <= 0) {
+        return 1;
+    }
+    lseek(fileno, chars, SEEK_CUR);
+
+    return 0;
+}
+
+int _skip_header(int fileno, int * version, int * flags)
+{
+    unsigned char r[4];
+    (void)read(fileno, r, 4);
+    unsigned char count = r[3];
+    *version = (r[0] & 0xff) << 8 | (r[1] & 0xff);
+    *flags = r[2];
+    lseek(fileno, (int)count, SEEK_CUR);
+    return 0;
+}
+
+long _read_word(int fileno)
+{
+    long w;
+    read(fileno, &w, WORD_SIZE);
+    return w;
+}
+
+void * _read_addr(int fileno)
+{
+    void * a;
+    read(fileno, &a, ADDR_SIZE);
+    return a;
+}
+
+int _skip_word(int fileno)
+{
+    lseek(fileno, WORD_SIZE, SEEK_CUR);
+    return 0;
+}
+
+int _skip_addr(int fileno)
+{
+    lseek(fileno, ADDR_SIZE, SEEK_CUR);
+    return 0;
+}
+
+int _skip_time_and_zone(int fileno)
+{
+    lseek(fileno, sizeof(int64_t)*2 + 8, SEEK_CUR);
+    return 0;
+}
+
+
+void dump_native_symbols(int fileno)
+{
+    // only call this function
+    off_t orig_pos, cur_pos;
+    char marker;
+    ssize_t count;
+    int version;
+    int flags;
+    int memory, lines, native;
+    orig_pos = lseek(fileno, 0, SEEK_CUR);
+
+    lseek(fileno, 5*WORD_SIZE, SEEK_SET);
+
+    while (1) {
+        LOG("pre read\n");
+        count = read(fileno, &marker, 1);
+        LOG("post read\n");
+        if (count <= 0) {
+            break;
+        }
+        cur_pos = lseek(fileno, 0, SEEK_CUR);
+        LOG("posss 0x%llx %d\n", cur_pos-1, cur_pos-1);
+        switch (marker) {
+            case MARKER_HEADER: {
+                LOG("header 0x%llx\n", cur_pos);
+                if (_skip_header(fileno, &version, &flags) != 0) {
+                    return;
+                }
+                memory = (flags & PROFILE_MEMORY) != 0;
+                native = (flags & PROFILE_NATIVE) != 0;
+                lines = (flags & PROFILE_LINES) != 0;
+                break;
+            } case MARKER_META: {
+                LOG("meta 0x%llx\n", cur_pos);
+                if (_skip_string(fileno) != 0) { return; }
+                if (_skip_string(fileno) != 0) { return; }
+                break;
+            } case MARKER_TIME_N_ZONE:
+              case MARKER_TRAILER: {
+                LOG("tnz or trailer 0x%llx\n", cur_pos);
+                if (_skip_time_and_zone(fileno) != 0) { return; }
+                break;
+            } case MARKER_VIRTUAL_IP:
+              case MARKER_NATIVE_SYMBOLS: {
+                LOG("virtip 0x%llx\n", cur_pos);
+                if (_skip_addr(fileno) != 0) { return; }
+                if (_skip_string(fileno) != 0) { return; }
+                break;
+            } case MARKER_STACKTRACE: {
+                long trace_count = _read_word(fileno);
+                long depth = _read_word(fileno);
+
+                LOG("stack 0x%llx %d %d\n", cur_pos, trace_count, depth);
+
+                for (long i = depth/2-1; i >= 0; i--) {
+                    long kind = (long)_read_addr(fileno);
+                    void * addr = _read_addr(fileno);
+                    if (kind == VMPROF_NATIVE_TAG) {
+                        LOG("found kind %p\n", addr);
+                        char name[MAXLEN];
+                        char srcfile[MAXLEN];
+                        name[0] = 0;
+                        srcfile[0] = 0;
+                        int lineno = 0;
+                        if (vmp_resolve_addr(addr, name, MAXLEN, &lineno, srcfile, MAXLEN) == 0) {
+                            LOG("dumping add %p, name %s, %s:%d\n", addr, name, srcfile, lineno);
+                            _dump_native_symbol(fileno, addr, name, lineno, srcfile);
+                        }
+                    }
+                }
+                LOG("passed  memory %d \n", memory);
+
+                if (_skip_addr(fileno) != 0) { return; } // thread id
+                if (memory) {
+                    if (_skip_addr(fileno) != 0) { return; } // profile memory
+                }
+
+                break;
+            } default: {
+                fprintf(stderr, "unknown marker 0x%x\n", marker);
+                return;
+            }
+        }
+
+        cur_pos = lseek(fileno, 0, SEEK_CUR);
+        if (cur_pos >= orig_pos) {
+            break;
+        }
+    }
+
+    lseek(fileno, 0, SEEK_END);
+}
+#endif
diff --git a/rpython/rlib/rvmprof/src/shared/trampoline.c b/rpython/rlib/rvmprof/src/shared/trampoline.c
--- a/rpython/rlib/rvmprof/src/shared/trampoline.c
+++ b/rpython/rlib/rvmprof/src/shared/trampoline.c
@@ -183,7 +183,7 @@
     int pagesize;
 
     if (g_trampoline != NULL) {
-        fprintf(stderr, "trampoline already patched\n");
+        //fprintf(stderr, "trampoline already patched\n");
         return 0; // already patched
     }
 
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -98,8 +98,6 @@
     result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
     *depth = *depth + 1;
 #else
-    //result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
-    //*depth = *depth + 1;
 
     if (frame->kind == VMPROF_CODE_TAG) {
         int n = *depth;
@@ -107,12 +105,7 @@
         result[n++] = (void*)frame->value;
         *depth = n;
     }
-#ifdef PYPY_JIT_CODEMAP
-    else if (frame->kind == VMPROF_JITTED_TAG) {
-        intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
-        n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
-    }
-#endif
+
 
 #endif
 
@@ -130,11 +123,15 @@
 
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
 int _write_native_stack(void* addr, void ** result, int depth) {
+#ifdef RPYTHON_VMPROF
+    result[depth++] = (void*)VMPROF_NATIVE_TAG;
+#else
     if (vmp_profiles_python_lines()) {
         // even if we do not log a python stack frame,
         // we must keep the profile readable
         result[depth++] = 0;
     }
+#endif
     result[depth++] = addr;
     return depth;
 }
@@ -143,16 +140,6 @@
 int vmp_walk_and_record_stack(PY_STACK_FRAME_T *frame, void ** result,
                               int max_depth, int native_skip, intptr_t pc) {
 
-//#ifdef PYPY_JIT_CODEMAP
-//    intptr_t codemap_addr;
-//    if (pypy_find_codemap_at_addr((intptr_t)pc, &codemap_addr)) {
-//        // the bottom part is jitted, means we can fill up the first part
-//        // from the JIT
-//        depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
-//        frame = FRAME_STEP(frame); // skip the first item as it contains garbage
-//    }
-//#endif
-
     // called in signal handler
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
     intptr_t func_addr;
@@ -179,12 +166,22 @@
         native_skip--;
     }
 
+    //printf("stack trace:\n");
     int depth = 0;
     PY_STACK_FRAME_T * top_most_frame = frame;
     while (depth < max_depth) {
         unw_get_proc_info(&cursor, &pip);
 
         func_addr = pip.start_ip;
+
+        //{
+        //    char name[64];
+        //    unw_word_t x;
+        //    unw_get_proc_name(&cursor, name, 64, &x);
+        //    printf("  %s %p\n", name, func_addr);
+        //}
+
+
         //if (func_addr == 0) {
         //    unw_word_t rip = 0;
         //    if (unw_get_reg(&cursor, UNW_REG_IP, &rip) < 0) {
@@ -197,6 +194,7 @@
 
         if ((void*)pip.start_ip == (void*)VMPROF_EVAL()) {
             // yes we found one stack entry of the python frames!
+#ifndef RPYTHON_VMPROF
             unw_word_t rbx = 0;
             if (unw_get_reg(&cursor, REG_RBX, &rbx) < 0) {
                 break;
@@ -208,6 +206,9 @@
                 // current top_most_frame
                 return 0;
             } else {
+#else
+            {
+#endif
                 if (top_most_frame == NULL) {
                     break;
                 }
@@ -222,13 +223,27 @@
             // mark native routines with the first bit set,
             // this is possible because compiler align to 8 bytes.
             //
-            depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
+
+#ifdef PYPY_JIT_CODEMAP
+            if (func_addr == 0 && top_most_frame->kind == VMPROF_JITTED_TAG) {
+                intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
+                n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
+                frame = FRAME_STEP(frame);
+            } else if (func_addr != 0) {
+                depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
+            }
+#else
+            if (func_addr != 0) {
+                depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
+            }
+#endif
         }
 
         int err = unw_step(&cursor);
-        if (err <= 0) {
-            // on mac this breaks on Py_Main?
+        if (err == 0) {
             break;
+        } else if (err < 0) {
+            return 0; // this sample is broken, cannot walk it fully
         }
     }
 
@@ -256,15 +271,25 @@
 int _ignore_symbols_from_path(const char * name) {
     // which symbols should not be considered while walking
     // the native stack?
+#ifdef RPYTHON_VMPROF
+    if (strstr(name, "libpypy-c.so") != NULL
+        || strstr(name, "pypy-c") != NULL
+        || strstr(name, "pypy") != NULL) {
+        printf("ignoring %s\n", name);
+        return 1;
+    }
+#else
+    // cpython
     if (strstr(name, "python") != NULL &&
-#ifdef __unix__
+#  ifdef __unix__
         strstr(name, ".so\n") == NULL
-#elif defined(__APPLE__)
+#  elif defined(__APPLE__)
         strstr(name, ".so") == NULL
-#endif
+#  endif
        ) {
         return 1;
     }
+#endif
     return 0;
 }
 
@@ -440,6 +465,9 @@
 }
 
 int vmp_ignore_ip(intptr_t ip) {
+    if (vmp_range_count == 0) {
+        return 0;
+    }
     int i = vmp_binary_search_ranges(ip, vmp_ranges, vmp_range_count);
     if (i == -1) {
         return 0;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <unistd.h>
+
 // common defines
 #define MARKER_STACKTRACE '\x01'
 #define MARKER_VIRTUAL_IP '\x02'
@@ -21,6 +23,7 @@
 #define PROFILE_MEMORY '\x01'
 #define PROFILE_LINES  '\x02'
 #define PROFILE_NATIVE '\x04'
+#define PROFILE_RPYTHON '\x08'
 
 #ifdef VMPROF_UNIX
 #define VMP_SUPPORTS_NATIVE_PROFILING
@@ -41,6 +44,9 @@
 #define PY_THREAD_STATE_T void
 #define FRAME_STEP(f) f->next
 #define FRAME_CODE(f) f->
+// Is there is a way to tell the compiler
+// that this prototype can have ANY return value. Just removing
+// the return type will default to int
 typedef long Signed;
 RPY_EXTERN Signed __vmprof_eval_vmprof();
 #define VMPROF_EVAL() __vmprof_eval_vmprof
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -106,6 +106,9 @@
     header.interp_name[2] = VERSION_TIMESTAMP;
     header.interp_name[3] = memory*PROFILE_MEMORY + lines*PROFILE_LINES + \
                             native*PROFILE_NATIVE;
+#ifdef RPYTHON_VMPROF
+    header.interp_name[3] += PROFILE_RPYTHON;
+#endif
     header.interp_name[4] = (char)namelen;
 
     memcpy(&header.interp_name[5], interp_name, namelen);
diff --git a/rpython/rlib/rvmprof/src/vmprof_stack.h b/rpython/rlib/rvmprof/src/vmprof_stack.h
--- a/rpython/rlib/rvmprof/src/vmprof_stack.h
+++ b/rpython/rlib/rvmprof/src/vmprof_stack.h
@@ -12,6 +12,7 @@
 #define VMPROF_JITTING_TAG 4
 #define VMPROF_GC_TAG 5
 #define VMPROF_ASSEMBLER_TAG 6
+#define VMPROF_NATIVE_TAG 7
 // whatever we want here
 
 typedef struct vmprof_stack_s {
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -154,7 +154,11 @@
     eci = ExternalCompilationInfo(compile_extra=['-g','-O2'],
             separate_module_sources=["""
             RPY_EXTERN int native_func(void) {
-                return 42;
+                int j = 0;
+                for (int i = 0; i < 420000; i++) {
+                    j += 1;
+                }
+                return j;
             }
             """])
 
@@ -176,8 +180,6 @@
         if num > 0:
             return main(code, num-1)
         else:
-            for i in range(100):
-                native_func()
             return native_func()
 
     tmpfilename = str(udir.join('test_rvmprof'))
@@ -188,15 +190,12 @@
             os.open('foo', 1, 1)
         code = MyCode()
         rvmprof.register_code(code, get_name)
-        fd = os.open(tmpfilename, os.O_WRONLY | os.O_CREAT, 0666)
-        if we_are_translated():
-            num = 100000000
-            period = 0.0001
-        else:
-            num = 10000
-            period = 0.9
+        fd = os.open(tmpfilename, os.O_RDWR | os.O_CREAT, 0666)
+        num = 10000
+        period = 0.0001
         rvmprof.enable(fd, period, native=1)
-        res = main(code, num)
+        for i in range(num):
+            res = main(code, 10)
         #assert res == 499999500000
         rvmprof.disable()
         os.close(fd)
@@ -204,10 +203,13 @@
 
     def check_profile(filename):
         from vmprof import read_profile
+        from vmprof.show import PrettyPrinter
 
         prof = read_profile(filename)
-        assert prof.get_tree().name.startswith("py:")
-        assert prof.get_tree().count
+        tree = prof.get_tree()
+        p = PrettyPrinter()
+        p._print_tree(tree)
+        assert tree.name.startswith("n:pypy_g_main")
 
     #assert f() == 0
     #assert os.path.exists(tmpfilename)

From pypy.commits at gmail.com  Wed Feb  8 13:53:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Feb 2017 10:53:56 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: Update comments with a plan
Message-ID: <589b6944.54b61c0a.ed992.97ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90011:f8ac0ceb09a5
Date: 2017-02-08 19:53 +0100
http://bitbucket.org/pypy/pypy/changeset/f8ac0ceb09a5/

Log:	Update comments with a plan

diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py
--- a/rpython/jit/backend/x86/guard_compat.py
+++ b/rpython/jit/backend/x86/guard_compat.py
@@ -9,38 +9,62 @@
 
 
 #
-# GUARD_COMPATIBLE(reg, const-ptr) produces the following assembler.
-# We also have the normal failure code at <failure_recovery>, which is
-# not put in the assembler but only in a field of the descr.  In the
-# following code, ofs(x) means the offset in the GC table of the
-# pointer 'x':
+# GUARD_COMPATIBLE(reg, const-ptr) produces the same assembler as
+# a GUARD_VALUE.  In the following code, ofs(x) means the offset in
+# the GC table of the pointer 'x':
 #
-#     MOV reg2, [RIP + ofs(_backend_choices)]    # LOAD_FROM_GC_TABLE
-#     CMP reg, [reg2 + bc_most_recent]
+#     MOV reg2, [RIP + ofs(const-ptr)]     # LOAD_FROM_GC_TABLE
+#     CMP reg, reg2
+#     JNE recovery_stub
+#   sequel:
+#     <reg2 not used any more>
+#
+# The difference is that 'recovery_stub' does not jump to one of the
+# 'failure_recovery_code' versions, but instead it jumps to
+# 'expand_guard_compatible'.  The latter calls invoke_find_compatible.
+# The result is one of:
+#
+#   * 0: bail out.  We jump to the 'failure_recovery_code'.
+#
+#   * -1: continue running on the same path.  We patch ofs(const-ptr)
+#     to contain the new value, and jump to 'sequel'.
+#
+#   * otherwise, it's the address of a bridge.  We jump to that bridge.
+#
+# This is the basic idea, but not the truth.  Things are more
+# complicated because we cache in the assembler the
+# invoke_find_compatible call results.  'expand_guard_compatible'
+# actually allocates a '_backend_choices' object, copies on it
+# various data it got from the recovery_stub, then patches the
+# recovery stub to this (the original recovery stub was padded if
+# necessary to have enough room):
+#
+#   recovery_stub:
+#     MOV R11, [RIP + ofs(_backend_choices)]
+#     CMP reg, [R11 + bc_most_recent]
 #     JNE slow_case
-#     JMP *[reg2 + bc_most_recent + 8]
+#     JMP *[R11 + bc_most_recent + 8]
 #   slow_case:
-#     PUSH RDX        # save
-#     PUSH RAX        # save
-#     MOV RDX=reg2, RAX=reg
-#            RDX is the _backend_choices object, RAX is the value to search for
-#     JMP search_tree    # see below
-#   sequel:
+#     PUSH RAX        # save the original value of RAX
+#     MOV RAX, reg    # the value to search for
+#     JMP *[R11 + bc_search_tree]    # x86-64: trick for a compact encoding
 #
-# The faildescr for this guard is a GuardCompatibleDescr.  We add to
-# them a few fields:
+# The faildescr for the GUARD_COMPATIBLE is a GuardCompatibleDescr.
+# Fields relevant for this discussion:
 #
-#     - _backend_choices_addr: points inside the GC table, to
-#                              ofs(_backend_choices)
-#     - _backend_sequel_label: points to the <sequel> label
-#     - _backend_failure_recovery: points to the <failure_recovery> label
-#     - _backend_gcmap: a copy of the gcmap at this point
+#     - _backend_ptr_addr: points inside the GC table, to ofs(const-ptr).
+#                          ofs(_backend_choices) is just afterwards.
+#                          Initially _backend_choices is NULL.
+#     - adr_jump_offset: raw address of the 'sequel' label (this field
+#                        is the same as on any other GuardDescr)
 #
-# The '_backend_choices' object itself is a separate GC struct/array
-# with the following fields:
+# The '_backend_choices' object itself, when allocated, is a separate
+# GC struct/array with the following fields:
 #
-#     - bc_faildescr: a copy of the faildescr of that guard
+#     - bc_faildescr: a reference to the faildescr of that guard
+#     - bc_gcmap: a copy of the gcmap at this point
 #     - bc_gc_table_tracer: only for a gc_writebarrier()
+#     - bc_search_tree: always the 'search_tree' label below
 #     - bc_most_recent: 1 pair (gcref, asmaddr)
 #     - bc_list: N pairs (gcref, asmaddr) sorted according to gcref
 #
@@ -48,66 +72,70 @@
 # gcrefs move, and ignores the tail of bc_list which contains the
 # invalid gcref of value -1.
 #
-# Initially, the _backend_choices contains a list of length 1, and
-# both bc_most_recent and bc_list[0] contain the same pair (gcref,
-# sequel), where 'gcref' is the 2nd argument to guard_compatible() and
-# <sequel> is the address of the label above.
-#
-# In general, the list can grow to contain all items for which
-# find_compatible() was called and returned non-zero.  Every entry
-# caches the result in 'asmaddr'.  The separate 'most_recent' entry
-# caches the last value seen, along with the result of
-# find_compatible().  If this find_compatible() returned zero, then
-# the cache entry contains the 'fail_guard' label below as the
-# 'asmaddr' value (such a value is never found inside bc_list, only in
-# bc_most_recent).
+# The bc_list can grow to contain all items for which find_compatible()
+# was called and returned non-zero.  Every entry caches the result in
+# 'asmaddr'.  The separate 'most_recent' entry caches the last value
+# seen, along with the result of find_compatible().
 #
 # The list is sorted, so we can search it using binary search.  The
 # length N of the list is equal to '2**D - 1', where D is the depth of
-# the tree algo.  Lists start with 1 item (D=1) and grow to the next
+# the tree algo.  Lists start with room for 3 items (D=2) and grow to the next
 # power-of-two-minus-one every time they need to be reallocated.  The
 # list is over-allocated, and the tail contains pairs (-1, ?), with -1
 # being the largest unsigned value (and never a valid GC pointer).
 #
-# When find_compatible() returns -1, we replace it with the address of
-# the 'sequel' label above, so that we don't have to special-case it
-# any more.  When find_compatible() returns 0, it is not stored in the
-# list, but still stored in bc_most_recent, with the 0 replaced with
-# the <failure_recovery> address introduced above.
+# When find_compatible() returns -1, we save instead the address of
+# 'sequel' in 'asmaddr'.  We could also patch ofs(const-ptr) so that the
+# fastest path applies if the same value is seen the next time; it
+# should be measured if it changes anything or not, because such a
+# patching occurs only once (i.e. 'search_tree' will not patch it
+# again).
+#
+# When find_compatible() returns 0, it is not stored in bc_list,
+# but still stored in bc_most_recent, with 'guard_compat_recovery'
+# as the 'asmaddr'.  Here is 'guard_compat_recovery': it emulates
+# generate_quick_failure() from assembler.py, and so it plays the role
+# of the original (patched) recovery stub.
+#
+#   guard_compat_recovery:
+#     PUSH R11
+#     PUSH [R11 + bc_gcmap]
+#     JMP target
 #
 # Here is the x86-64 runtime code to walk the tree:
 #
 #   search_tree:
-#     MOV [RSP+16], RDX                    # save
-#     MOV R11, [RDX + bc_list.length]      # a power of two minus one
-#     ADD RDX, $bc_list.items
+#     MOV [ESP+8], RCX                     # save the original value
+#     MOV [ESP+16], R11                    # save the _backend_choices object
+#     MOV RCX, [R11 + bc_list.length]      # a power of two minus one
+#     ADD R11, $bc_list.items
 #     JMP loop
 #
 #   right:
-#     LEA RDX, [RDX + 8*R11 + 8]
+#     LEA R11, [R11 + 8*RCX + 8]
 #   left:
-#     SHR R11, 1
+#     SHR RCX, 1
 #     JZ not_found
 #   loop:
-#     # search for the item at addresses between RDX and RDX+16*R11, included
-#     # (note that R11 is always odd here; even though we use 8*R11 in the
+#     # search for the item at addresses between R11 and R11+16*RCX, included
+#     # (note that RCX is always odd here; even though we use 8*RCX in the
 #     # following instruction, we're really accessing 16-bytes-sized items)
-#     CMP RAX, [RDX + 8*R11 - 8]      # R11 = ...31, 15, 7, 3, 1
+#     CMP RAX, [R11 + 8*RCX - 8]      # RCX = ...31, 15, 7, 3, 1
 #     JA right
 #     JNE left
 #
 #   found:
-#     MOV R11, [RDX + 8*R11]
-#     MOV RDX, [RSP+16]
-#     MOV [RDX + bc_most_recent], RAX
-#     MOV [RDX + bc_most_recent + 8], R11
-#     POP RAX
-#     POP RDX
+#     MOV R11, [R11 + 8*RCX]             # address to jump to next
+#     MOV RCX, [ESP+16]                  # reload the _backend_choices object
+#     MOV [RCX + bc_most_recent], RAX
+#     MOV [RCX + bc_most_recent + 8], R11
+#     MOV RCX, [ESP+8]                   # restore saved value
+#     POP RAX                            # pushed by the caller
 #     JMP *R11
 #
 #   not_found:
 #     <save all registers to the jitframe RBP,
-#         reading and popping the original RAX and RDX off the stack>
+#         reading and popping the original RAX and RCX off the stack>
 #     <call invoke_find_compatible(_backend_choices=[RSP], value=RAX),
 #                                  jitframe=RBP>
 #     <_reload_frame_if_necessary>
@@ -147,25 +175,6 @@
 #
 # ____________________________________________________________
 
-# Possible optimization: GUARD_COMPATIBLE(reg, const-ptr) could emit
-# first assembler that is similar to a GUARD_VALUE.  As soon as a
-# second case is seen, this assembler is patched (once) to turn it
-# into the general switching version above.  The entry in the GC table
-# at ofs(_backend_choices) starts with the regular const-ptr, and the
-# BACKEND_CHOICES object is only allocated when the assembler is
-# patched.  The original assembler can be similar to a GUARD_VALUE:
-#
-#     MOV reg2, [RIP + ofs(const-ptr)]    # == ofs(_backend_choices)
-#     CMP reg, reg2
-#     JE sequel
-#     PUSH [RIP + ofs(guard_compatible_descr)]
-#     JMP guard_compat_second_case
-#     <padding to make the code large enough for patching>
-#     <ends with one byte which tells the size of this block>
-#   sequel:
-#
-# ____________________________________________________________
-
 
 # A lot of the logic is not specific to the x86 backend and is
 # written in ../llsupport/guard_compat.py.

From pypy.commits at gmail.com  Wed Feb  8 15:41:00 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Feb 2017 12:41:00 -0800 (PST)
Subject: [pypy-commit] pypy buffer-cleanup: Close branch buffer-cleanup
Message-ID: <589b825c.cc881c0a.e578d.be15@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: buffer-cleanup
Changeset: r90012:ae3f90797917
Date: 2017-02-08 20:40 +0000
http://bitbucket.org/pypy/pypy/changeset/ae3f90797917/

Log:	Close branch buffer-cleanup


From pypy.commits at gmail.com  Wed Feb  8 15:41:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Feb 2017 12:41:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Merged in buffer-cleanup (pull request
 #515)
Message-ID: <589b826d.ccaddf0a.19ca1.285b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90013:3059a0362786
Date: 2017-02-08 20:40 +0000
http://bitbucket.org/pypy/pypy/changeset/3059a0362786/

Log:	Merged in buffer-cleanup (pull request #515)

	Buffer cleanup

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -25,15 +25,6 @@
                                       reds=['items', 'w_iterator'])
 
 
- at specialize.memo()
-def _does_override_buffer_w(type):
-    return type.buffer_w != W_Root.buffer_w
-
- at specialize.memo()
-def _does_override_buffer_w_ex(type):
-    return type.buffer_w_ex != W_Root.buffer_w_ex
-
-
 class W_Root(object):
     """This is the abstract root class of all wrapped objects that live
     in a 'normal' object space like StdObjSpace."""
@@ -223,15 +214,8 @@
         return None
 
     def buffer_w(self, space, flags):
-        if _does_override_buffer_w_ex(self.__class__):
-            return self.buffer_w_ex(space, flags)[0]
         return self.__buffer_w(space, flags).buffer_w(space, flags)
 
-    def buffer_w_ex(self, space, flags):
-        if _does_override_buffer_w(self.__class__):
-            return self.buffer_w(space, flags), 'B', 1
-        return self.__buffer_w(space, flags).buffer_w_ex(space, flags)
-
     def __buffer_w(self, space, flags):
         if flags & space.BUF_WRITABLE:
             w_impl = space.lookup(self, '__wbuffer__')
@@ -1469,15 +1453,6 @@
             raise oefmt(self.w_TypeError,
                         "'%T' does not support the buffer interface", w_obj)
 
-    def buffer_w_ex(self, w_obj, flags):
-        # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
-        # Returns extra information: (buffer, typecode, itemsize)
-        try:
-            return w_obj.buffer_w_ex(self, flags)
-        except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "'%T' does not support the buffer interface", w_obj)
-
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -91,6 +91,8 @@
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
+        self.fmt = shape.itemcode
+        self.itemsize = shape.size
 
     def descr_repr(self, space):
         addr = rffi.cast(lltype.Unsigned, self.ll_buffer)
@@ -105,8 +107,7 @@
             raise segfault_exception(space, "setting element of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        unwrap_value(space, write_ptr, self.ll_buffer, num,
-                     self.shape.itemcode, w_value)
+        unwrap_value(space, write_ptr, self.ll_buffer, num, self.fmt, w_value)
 
     def descr_setitem(self, space, w_index, w_value):
         try:
@@ -123,8 +124,7 @@
             raise segfault_exception(space, "accessing elements of freed array")
         if num >= self.length or num < 0:
             raise OperationError(space.w_IndexError, space.w_None)
-        return wrap_value(space, read_ptr, self.ll_buffer, num,
-                          self.shape.itemcode)
+        return wrap_value(space, read_ptr, self.ll_buffer, num, self.fmt)
 
     def descr_getitem(self, space, w_index):
         try:
@@ -141,19 +141,16 @@
 
     @unwrap_spec(num=int)
     def descr_itemaddress(self, space, num):
-        itemsize = self.shape.size
-        ptr = rffi.ptradd(self.ll_buffer, itemsize * num)
+        ptr = rffi.ptradd(self.ll_buffer, self.itemsize * num)
         return space.wrap(rffi.cast(lltype.Unsigned, ptr))
 
     def getrawsize(self):
-        itemsize = self.shape.size
-        return itemsize * self.length
+        return self.itemsize * self.length
 
     def decodeslice(self, space, w_slice):
         if not space.isinstance_w(w_slice, space.w_slice):
             raise oefmt(space.w_TypeError, "index must be int or slice")
-        letter = self.shape.itemcode
-        if letter != 'c':
+        if self.fmt != 'c':
             raise oefmt(space.w_TypeError, "only 'c' arrays support slicing")
         w_start = space.getattr(w_slice, space.wrap('start'))
         w_stop = space.getattr(w_slice, space.wrap('stop'))
@@ -192,9 +189,6 @@
         for i in range(len(value)):
             ll_buffer[start + i] = value[i]
 
-    def buffer_w_ex(self, space, flags):
-        return self.buffer_w(space, flags), self.shape.itemcode, self.shape.size
-
 
 W_ArrayInstance.typedef = TypeDef(
     'ArrayInstance',
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -14,6 +14,12 @@
     def getlength(self):
         return self.datainstance.getrawsize()
 
+    def getformat(self):
+        return self.datainstance.fmt
+
+    def getitemsize(self):
+        return self.datainstance.itemsize
+
     def getitem(self, index):
         ll_buffer = self.datainstance.ll_buffer
         return ll_buffer[index]
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -337,6 +337,8 @@
 
 
 class W_DataInstance(W_Root):
+    fmt = 'B'
+    itemsize = 1
     def __init__(self, space, size, address=r_uint(0)):
         if address:
             self.ll_buffer = rffi.cast(rffi.VOIDP, address)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -256,8 +256,8 @@
         if oldbuffer:
             lltype.free(oldbuffer, flavor='raw')
 
-    def buffer_w_ex(self, space, flags):
-        return ArrayBuffer(self, False), self.typecode, self.itemsize
+    def buffer_w(self, space, flags):
+        return ArrayBuffer(self, False)
 
     def descr_append(self, space, w_x):
         """ append(x)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,10 +1,10 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP,
+    cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP, cts,
     generic_cpy_call,
     PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES, PyBUF_SIMPLE)
-from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
+from pypy.module.cpyext.pyobject import PyObject, incref, Py_DecRef
 
 @cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
 def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
@@ -35,6 +35,33 @@
         Py_DecRef(space, view.c_obj)
     return 0
 
+def fill_buffer(space, view, pybuf, py_obj):
+    view.c_buf = cts.cast('void *', pybuf.get_raw_address())
+    view.c_obj = py_obj
+    if py_obj:
+        incref(space, py_obj)
+    view.c_len = pybuf.getlength()
+    view.c_itemsize = pybuf.getitemsize()
+    rffi.setintfield(view, 'c_readonly', int(pybuf.readonly))
+    rffi.setintfield(view, 'c_ndim', pybuf.getndim())
+    view.c_format = rffi.str2charp(pybuf.getformat())
+    shape = pybuf.getshape()
+    if not shape:
+        view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+    else:
+        view.c_shape = cts.cast('Py_ssize_t*', view.c__shape)
+        for i, n in enumerate(shape):
+            view.c_shape[i] = n
+    strides = pybuf.getstrides()
+    if not strides:
+        view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    else:
+        view.c_strides = cts.cast('Py_ssize_t*', view.c__strides)
+        for i, n in enumerate(strides):
+            view.c_strides[i] = n
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
 
 @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
               lltype.Signed, lltype.Signed], rffi.INT, error=-1)
@@ -51,7 +78,7 @@
     view.c_len = length
     view.c_obj = obj
     if obj:
-        Py_IncRef(space, obj)
+        incref(space, obj)
     view.c_itemsize = 1
     rffi.setintfield(view, 'c_readonly', readonly)
     rffi.setintfield(view, 'c_ndim', 1)
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2871,7 +2871,17 @@
     return PyBytes_FromStringAndSize((char*)ptr, size);
 }
 
-
+static PyObject *
+write_buffer_len(PyObject * self, PyObject * obj)
+{
+    void* buf;
+    Py_ssize_t buf_len;
+    if (PyObject_AsWriteBuffer(obj, &buf, &buf_len) < 0) {
+        PyErr_SetString(PyExc_ValueError, "bad value");
+        return NULL;
+    }
+    return PyLong_FromLong(buf_len);
+}
 
 /*********************** Install Module **************************/
 
@@ -2880,6 +2890,7 @@
      PyDoc_STR("Internal. Used for pickling support.")},
     {"switch_multiply",   (PyCFunction)switch_multiply, METH_NOARGS, NULL},
     {"readbuffer_as_string",   (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL},
+    {"write_buffer_len", write_buffer_len, METH_O, NULL},
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -84,6 +84,9 @@
         arr = module.array('u', '123')
         view = memoryview(arr)
         assert view.itemsize == 4
+        assert module.write_buffer_len(arr) == 12
+        assert len(module.readbuffer_as_string(arr)) == 12
+        assert len(module.readbuffer_as_string(view)) == 12
 
     def test_subclass(self):
         import struct
diff --git a/pypy/module/cpyext/test/test_buffer.py b/pypy/module/cpyext/test/test_buffer.py
--- a/pypy/module/cpyext/test/test_buffer.py
+++ b/pypy/module/cpyext/test/test_buffer.py
@@ -1,5 +1,23 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+class AppTestBuffer(AppTestCpythonExtensionBase):
+    def test_AsWriteBuffer(self):
+        import array
+        module = self.import_extension('buffer', [
+            ('write_buffer_len', 'METH_O',
+             """
+             void* buf;
+             Py_ssize_t buf_len;
+             if (PyObject_AsWriteBuffer(args, &buf, &buf_len) < 0) {
+                PyErr_SetString(PyExc_ValueError, "bad value");
+                return NULL;
+             }
+             return PyLong_FromLong(buf_len);
+             """)])
+        assert module.write_buffer_len(bytearray(b'123')) == 3
+        assert module.write_buffer_len(array.array('i', [1, 2, 3])) == 12
+
+
 class AppTestMmap(AppTestCpythonExtensionBase):
     def test_mmap_buffer(self):
         module = self.import_extension('mmap_buffer', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -555,11 +555,10 @@
 
 @slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
 def bf_getbuffer(space, w_obj, view, flags):
-    from pypy.module.cpyext.buffer import PyBuffer_FillInfo
+    from pypy.module.cpyext.buffer import fill_buffer
     buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
-    c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-    return PyBuffer_FillInfo(space, view, w_obj, c_buf,
-                             space.len_w(w_obj), 0, flags)
+    fill_buffer(space, view, buf, as_pyobj(space, w_obj))
+    return 0
 
 def setup_buffer_procs(space, w_type, pto):
     bufspec = w_type.layout.typedef.buffer
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -74,17 +74,18 @@
     def setformat(self, value):
         self.format = value
 
-    def buffer_w_ex(self, space, flags):
+    def buffer_w(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf, self.getformat(), self.itemsize
+        return self.buf
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
         if isinstance(w_object, W_MemoryView):
             w_object._check_released(space)
             return W_MemoryView.copy(w_object)
-        return W_MemoryView(*space.buffer_w_ex(w_object, space.BUF_FULL_RO))
+        buf = space.buffer_w(w_object, space.BUF_FULL_RO)
+        return W_MemoryView(buf, buf.getformat(), buf.getitemsize())
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -345,9 +345,6 @@
         return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
                           self.w_size, self.w_strides, self.w_shape)
 
-    def buffer_w_ex(self, space, flags):
-        return self.buffer_w(space, flags), space.str_w(self.w_fmt), space.int_w(self.w_size)
-
 W_MockArray.typedef = TypeDef("MockArray",
     __new__ = interp2app(W_MockArray.descr_new),
 )

From pypy.commits at gmail.com  Wed Feb  8 23:16:27 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Wed, 08 Feb 2017 20:16:27 -0800 (PST)
Subject: [pypy-commit] pypy default: Bump greenlet version
Message-ID: <589bed1b.ccaddf0a.19ca1.8be7@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r90014:fef27077c286
Date: 2017-02-08 23:15 -0500
http://bitbucket.org/pypy/pypy/changeset/fef27077c286/

Log:	Bump greenlet version

diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.11
+Version: 0.4.12
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.11"
+__version__ = "0.4.12"
 
 # ____________________________________________________________
 # Exceptions

From pypy.commits at gmail.com  Thu Feb  9 01:46:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Feb 2017 22:46:32 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Slides, same as for pyconza2016
Message-ID: <589c1048.04abdf0a.22687.ae33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5775:a242a484e2ab
Date: 2017-02-09 07:41 +0100
http://bitbucket.org/pypy/extradoc/changeset/a242a484e2ab/

Log:	Slides, same as for pyconza2016

diff --git a/talk/swisspython2017/abstract.txt b/talk/swisspython2017/abstract.txt
--- a/talk/swisspython2017/abstract.txt
+++ b/talk/swisspython2017/abstract.txt
@@ -2,7 +2,7 @@
 --------
 
 RevDB is an experimental "reverse debugger" for Python, similar to
-UndoDB-GDB or LL for C. You run your program once, in "record" mode,
+UndoDB-GDB or RR for C. You run your program once, in "record" mode,
 producing a log file; once you get buggy behavior, you start the
 reverse-debugger on the log file. It gives an (improved) pdb-like
 experience, but it is replaying your program exactly as it ran---all
diff --git a/talk/pyconza2016/revdb/Makefile b/talk/swisspython2017/revdb/Makefile
copy from talk/pyconza2016/revdb/Makefile
copy to talk/swisspython2017/revdb/Makefile
diff --git a/talk/pyconza2016/revdb/author.latex b/talk/swisspython2017/revdb/author.latex
copy from talk/pyconza2016/revdb/author.latex
copy to talk/swisspython2017/revdb/author.latex
--- a/talk/pyconza2016/revdb/author.latex
+++ b/talk/swisspython2017/revdb/author.latex
@@ -6,5 +6,5 @@
 \ \ \ \ \ \ \ \ \ \includegraphics[scale=0.05]{pypylogo.png}}
 
 
-\institute{PyCon ZA 2016}
-\date{October 2016}
+\institute{Swiss Python Summit 2017}
+\date{February 2017}
diff --git a/talk/pyconza2016/revdb/demo/demo1.py b/talk/swisspython2017/revdb/demo/demo1.py
copy from talk/pyconza2016/revdb/demo/demo1.py
copy to talk/swisspython2017/revdb/demo/demo1.py
diff --git a/talk/pyconza2016/revdb/demo/todo.txt b/talk/swisspython2017/revdb/demo/todo.txt
copy from talk/pyconza2016/revdb/demo/todo.txt
copy to talk/swisspython2017/revdb/demo/todo.txt
diff --git a/talk/pyconza2016/revdb/stylesheet.latex b/talk/swisspython2017/revdb/stylesheet.latex
copy from talk/pyconza2016/revdb/stylesheet.latex
copy to talk/swisspython2017/revdb/stylesheet.latex
diff --git a/talk/pyconza2016/revdb/talk.pdf b/talk/swisspython2017/revdb/talk.pdf
copy from talk/pyconza2016/revdb/talk.pdf
copy to talk/swisspython2017/revdb/talk.pdf
index 51c4b5737abc9af5a4d043416a98f1afae522e8d..936eb31c60797aab7e3b6d9bb269723da9440b49
GIT binary patch

[cut]

diff --git a/talk/pyconza2016/revdb/talk.rst b/talk/swisspython2017/revdb/talk.rst
copy from talk/pyconza2016/revdb/talk.rst
copy to talk/swisspython2017/revdb/talk.rst
--- a/talk/pyconza2016/revdb/talk.rst
+++ b/talk/swisspython2017/revdb/talk.rst
@@ -6,7 +6,7 @@
 Introduction
 ===========================
 
-* I am Armin Rigo, part of the PyPy project since 13 years
+* I am Armin Rigo, part of the PyPy project since the start (14 years)
 
   * PyPy is another implementation of Python
 
@@ -92,9 +92,9 @@
 
 * Track multiple processes
 
-* Windows (for $?)
+* Windows (contract possible)
 
-* Python 3 (soon?)
+* Python 3 (soon)
 
 
 Comparison

From pypy.commits at gmail.com  Thu Feb  9 01:46:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Feb 2017 22:46:34 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: merge heads
Message-ID: <589c104a.2a99df0a.665cb.b46f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5776:b8a362c95c2d
Date: 2017-02-09 07:46 +0100
http://bitbucket.org/pypy/extradoc/changeset/b8a362c95c2d/

Log:	merge heads

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -11,6 +11,7 @@
 ==================== ============== =======================
 Armin Rigo                          private
 Richard Plangger     26.02/04.02    Ermina
+Remi Meier           (?)27.02/04.02 Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Thu Feb  9 01:57:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Feb 2017 22:57:36 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: include the text of the demo
 inside the pdf, just in case
Message-ID: <589c12e0.0fb81c0a.47e09.4d94@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5777:417f4214a5b6
Date: 2017-02-09 07:57 +0100
http://bitbucket.org/pypy/extradoc/changeset/417f4214a5b6/

Log:	include the text of the demo inside the pdf, just in case

diff --git a/talk/swisspython2017/revdb/demo/_demo1.png b/talk/swisspython2017/revdb/demo/_demo1.png
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..fa6836abd3a312117a6bdef3d42cc70cc2ebec40
GIT binary patch

[cut]

diff --git a/talk/swisspython2017/revdb/talk.pdf b/talk/swisspython2017/revdb/talk.pdf
index 936eb31c60797aab7e3b6d9bb269723da9440b49..0b5bbf46bda5af01898f6b0594e0d5dd76055487
GIT binary patch

[cut]

diff --git a/talk/swisspython2017/revdb/talk.rst b/talk/swisspython2017/revdb/talk.rst
--- a/talk/swisspython2017/revdb/talk.rst
+++ b/talk/swisspython2017/revdb/talk.rst
@@ -21,6 +21,13 @@
 * Demo
 
 
+What is a reverse debugger?
+===========================
+
+.. image:: demo/_demo1.png
+   :scale: 25%
+
+
 How is that possible??
 ======================
 

From pypy.commits at gmail.com  Thu Feb  9 03:28:58 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 09 Feb 2017 00:28:58 -0800 (PST)
Subject: [pypy-commit] stmgc default: merge new finalizer API
Message-ID: <589c284a.48301c0a.b5123.6cb8@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2005:3af462fd449b
Date: 2017-02-09 09:28 +0100
http://bitbucket.org/pypy/stmgc/changeset/3af462fd449b/

Log:	merge new finalizer API

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1047,8 +1047,7 @@
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
-    assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
-    assert(STM_PSEGMENT->finalizers == NULL);
+    assert(list_is_empty(STM_PSEGMENT->young_objects_with_destructors));
     assert(STM_PSEGMENT->active_queues == NULL);
 #ifndef NDEBUG
     /* this should not be used when objects_pointing_to_nursery == NULL */
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -13,6 +13,8 @@
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
+#include <stdbool.h>
+#include "list.h"
 
 /************************************************************/
 
@@ -144,9 +146,9 @@
     pthread_t running_pthread;
 #endif
 
-    /* light finalizers */
-    struct list_s *young_objects_with_light_finalizers;
-    struct list_s *old_objects_with_light_finalizers;
+    /* destructors */
+    struct list_s *young_objects_with_destructors;
+    struct list_s *old_objects_with_destructors;
 
     /* regular finalizers (objs from the current transaction only) */
     struct finalizers_s *finalizers;
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -2,31 +2,65 @@
 # error "must be compiled via stmgc.c"
 # include "core.h"  // silence flymake
 #endif
+#include "finalizer.h"
+#include "fprintcolor.h"
+#include "nursery.h"
+#include "gcpage.h"
 
 /* callbacks */
-void (*stmcb_light_finalizer)(object_t *);
+void (*stmcb_destructor)(object_t *);
 void (*stmcb_finalizer)(object_t *);
 
 
 static void init_finalizers(struct finalizers_s *f)
 {
     f->objects_with_finalizers = list_create();
-    f->count_non_young = 0;
-    f->run_finalizers = NULL;
-    f->running_next = NULL;
+    f->probably_young_objects_with_finalizers = list_create();
+    f->run_finalizers = list_create();
+    f->lock = 0;
+    f->running_trigger_now = NULL;
 }
 
 static void setup_finalizer(void)
 {
     init_finalizers(&g_finalizers);
+
+    for (long j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+
+        assert(pseg->finalizers == NULL);
+        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
+        if (f == NULL)
+            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
+        init_finalizers(f);
+        pseg->finalizers = f;
+    }
 }
 
-static void teardown_finalizer(void)
+void stm_setup_finalizer_queues(int number, stm_finalizer_trigger_fn *triggers)
 {
-    if (g_finalizers.run_finalizers != NULL)
-        list_free(g_finalizers.run_finalizers);
-    list_free(g_finalizers.objects_with_finalizers);
+    assert(g_finalizer_triggers.count == 0);
+    assert(g_finalizer_triggers.triggers == NULL);
+
+    g_finalizer_triggers.count = number;
+    g_finalizer_triggers.triggers = (stm_finalizer_trigger_fn *)
+        malloc(number * sizeof(stm_finalizer_trigger_fn));
+
+    for (int qindex = 0; qindex < number; qindex++) {
+        g_finalizer_triggers.triggers[qindex] = triggers[qindex];
+        dprintf(("setup_finalizer_queue(qindex=%d,fun=%p)\n", qindex, triggers[qindex]));
+    }
+}
+
+static void teardown_finalizer(void) {
+    LIST_FREE(g_finalizers.run_finalizers);
+    LIST_FREE(g_finalizers.objects_with_finalizers);
+    LIST_FREE(g_finalizers.probably_young_objects_with_finalizers);
     memset(&g_finalizers, 0, sizeof(g_finalizers));
+
+    if (g_finalizer_triggers.triggers)
+        free(g_finalizer_triggers.triggers);
+    memset(&g_finalizer_triggers, 0, sizeof(g_finalizer_triggers));
 }
 
 static void _commit_finalizers(void)
@@ -36,36 +70,31 @@
         stm_spin_loop();
     }
 
-    if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
+    struct finalizers_s *local_fs = STM_PSEGMENT->finalizers;
+    if (!list_is_empty(local_fs->run_finalizers)) {
         /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
            'g_finalizers.run_finalizers', dropping any initial NULLs
            (finalizers already called) */
-        struct list_s *src = STM_PSEGMENT->finalizers->run_finalizers;
-        uintptr_t frm = 0;
-        if (STM_PSEGMENT->finalizers->running_next != NULL) {
-            frm = *STM_PSEGMENT->finalizers->running_next;
-            assert(frm <= list_count(src));
-            *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
-        }
-        if (frm < list_count(src)) {
-            if (g_finalizers.run_finalizers == NULL)
-                g_finalizers.run_finalizers = list_create();
+        struct list_s *src = local_fs->run_finalizers;
+        if (list_count(src)) {
             g_finalizers.run_finalizers = list_extend(
                 g_finalizers.run_finalizers,
-                src, frm);
+                src, 0);
         }
-        list_free(src);
     }
+    LIST_FREE(local_fs->run_finalizers);
 
     /* copy the whole 'STM_PSEGMENT->finalizers->objects_with_finalizers'
        into 'g_finalizers.objects_with_finalizers' */
     g_finalizers.objects_with_finalizers = list_extend(
         g_finalizers.objects_with_finalizers,
-        STM_PSEGMENT->finalizers->objects_with_finalizers, 0);
-    list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
+        local_fs->objects_with_finalizers, 0);
+    LIST_FREE(local_fs->objects_with_finalizers);
+    assert(list_is_empty(local_fs->probably_young_objects_with_finalizers));
+    LIST_FREE(local_fs->probably_young_objects_with_finalizers);
 
-    free(STM_PSEGMENT->finalizers);
-    STM_PSEGMENT->finalizers = NULL;
+    // re-init
+    init_finalizers(local_fs);
 
     __sync_lock_release(&g_finalizers.lock);
 }
@@ -74,24 +103,22 @@
 {
     /* like _commit_finalizers(), but forget everything from the
        current transaction */
-    if (pseg->finalizers != NULL) {
-        if (pseg->finalizers->run_finalizers != NULL) {
-            if (pseg->finalizers->running_next != NULL) {
-                *pseg->finalizers->running_next = (uintptr_t)-1;
-            }
-            list_free(pseg->finalizers->run_finalizers);
-        }
-        list_free(pseg->finalizers->objects_with_finalizers);
-        free(pseg->finalizers);
-        pseg->finalizers = NULL;
-    }
+    LIST_FREE(pseg->finalizers->run_finalizers);
+    LIST_FREE(pseg->finalizers->objects_with_finalizers);
+    LIST_FREE(pseg->finalizers->probably_young_objects_with_finalizers);
+    // re-init
+    init_finalizers(pseg->finalizers);
+
+    // if we were running triggers, release the lock:
+    if (g_finalizers.running_trigger_now == pseg)
+        g_finalizers.running_trigger_now = NULL;
 
     /* call the light finalizers for objects that are about to
        be forgotten from the current transaction */
     char *old_gs_register = STM_SEGMENT->segment_base;
     bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
 
-    struct list_s *lst = pseg->young_objects_with_light_finalizers;
+    struct list_s *lst = pseg->young_objects_with_destructors;
     long i, count = list_count(lst);
     if (lst > 0) {
         for (i = 0; i < count; i++) {
@@ -101,15 +128,15 @@
                 set_gs_register(pseg->pub.segment_base);
                 must_fix_gs = false;
             }
-            stmcb_light_finalizer(obj);
+            stmcb_destructor(obj);
         }
         list_clear(lst);
     }
 
     /* also deals with overflow objects: they are at the tail of
-       old_objects_with_light_finalizers (this list is kept in order
+       old_objects_with_destructors (this list is kept in order
        and we cannot add any already-committed object) */
-    lst = pseg->old_objects_with_light_finalizers;
+    lst = pseg->old_objects_with_destructors;
     count = list_count(lst);
     while (count > 0) {
         object_t *obj = (object_t *)list_item(lst, --count);
@@ -120,7 +147,7 @@
             set_gs_register(pseg->pub.segment_base);
             must_fix_gs = false;
         }
-        stmcb_light_finalizer(obj);
+        stmcb_destructor(obj);
     }
 
     if (STM_SEGMENT->segment_base != old_gs_register)
@@ -128,44 +155,42 @@
 }
 
 
-void stm_enable_light_finalizer(object_t *obj)
+void stm_enable_destructor(object_t *obj)
 {
     if (_is_young(obj)) {
-        LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->young_objects_with_destructors, obj);
     }
     else {
         assert(_is_from_same_transaction(obj));
-        LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_destructors, obj);
     }
 }
 
-object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
+
+void stm_enable_finalizer(int queue_index, object_t *obj)
 {
-    object_t *obj = _stm_allocate_external(size_rounded_up);
-
-    if (STM_PSEGMENT->finalizers == NULL) {
-        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
-        if (f == NULL)
-            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
-        init_finalizers(f);
-        STM_PSEGMENT->finalizers = f;
+    if (_is_young(obj)) {
+        LIST_APPEND(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers, queue_index);
     }
-    assert(STM_PSEGMENT->finalizers->count_non_young
-           <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers));
-    LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
-    return obj;
+    else {
+        assert(_is_from_same_transaction(obj));
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, queue_index);
+    }
 }
 
 
+
 /************************************************************/
-/*  Light finalizers
+/*  Destructors
 */
 
-static void deal_with_young_objects_with_finalizers(void)
+static void deal_with_young_objects_with_destructors(void)
 {
-    /* for light finalizers: executes finalizers for objs that don't survive
+    /* for destructors: executes destructors for objs that don't survive
        this minor gc */
-    struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
+    struct list_s *lst = STM_PSEGMENT->young_objects_with_destructors;
     long i, count = list_count(lst);
     for (i = 0; i < count; i++) {
         object_t *obj = (object_t *)list_item(lst, i);
@@ -174,28 +199,29 @@
         object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
         if (pforwarded_array[0] != GCWORD_MOVED) {
             /* not moved: the object dies */
-            stmcb_light_finalizer(obj);
+            stmcb_destructor(obj);
         }
         else {
             obj = pforwarded_array[1]; /* moved location */
             assert(!_is_young(obj));
-            LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+            LIST_APPEND(STM_PSEGMENT->old_objects_with_destructors, obj);
         }
     }
     list_clear(lst);
 }
 
-static void deal_with_old_objects_with_finalizers(void)
+static void deal_with_old_objects_with_destructors(void)
 {
-    /* for light finalizers */
+    /* for destructors */
     int old_gs_register = STM_SEGMENT->segment_num;
     int current_gs_register = old_gs_register;
     long j;
-    assert(list_is_empty(get_priv_segment(0)->old_objects_with_light_finalizers));
+    assert(list_is_empty(get_priv_segment(0)->old_objects_with_destructors));
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
 
-        struct list_s *lst = pseg->old_objects_with_light_finalizers;
+        assert(list_is_empty(pseg->young_objects_with_destructors));
+        struct list_s *lst = pseg->old_objects_with_destructors;
         long i, count = list_count(lst);
         lst->count = 0;
         for (i = 0; i < count; i++) {
@@ -217,7 +243,7 @@
                     set_gs_register(get_segment_base(j));
                     current_gs_register = j;
                 }
-                stmcb_light_finalizer(obj);
+                stmcb_destructor(obj);
             }
             else {
                 /* object survives */
@@ -230,6 +256,7 @@
 }
 
 
+
 /************************************************************/
 /*  Algorithm for regular (non-light) finalizers.
     Follows closely pypy/doc/discussion/finalizer-order.rst
@@ -328,20 +355,23 @@
 
     struct list_s *marked = list_create();
 
+    assert(list_is_empty(f->probably_young_objects_with_finalizers));
     struct list_s *lst = f->objects_with_finalizers;
     long i, count = list_count(lst);
     lst->count = 0;
-    f->count_non_young = 0;
 
-    for (i = 0; i < count; i++) {
+    for (i = 0; i < count; i += 2) {
         object_t *x = (object_t *)list_item(lst, i);
+        uintptr_t qindex = list_item(lst, i + 1);
 
         assert(_finalization_state(x) != 1);
         if (_finalization_state(x) >= 2) {
             list_set_item(lst, lst->count++, (uintptr_t)x);
+            list_set_item(lst, lst->count++, qindex);
             continue;
         }
         LIST_APPEND(marked, x);
+        LIST_APPEND(marked, qindex);
 
         struct list_s *pending = _finalizer_pending;
         LIST_APPEND(pending, x);
@@ -373,27 +403,29 @@
     struct list_s *run_finalizers = f->run_finalizers;
 
     long i, count = list_count(marked);
-    for (i = 0; i < count; i++) {
+    for (i = 0; i < count; i += 2) {
         object_t *x = (object_t *)list_item(marked, i);
+        uintptr_t qindex = list_item(marked, i + 1);
 
         int state = _finalization_state(x);
         assert(state >= 2);
         if (state == 2) {
-            if (run_finalizers == NULL)
-                run_finalizers = list_create();
             LIST_APPEND(run_finalizers, x);
+            LIST_APPEND(run_finalizers, qindex);
             _recursively_bump_finalization_state_from_2_to_3(pseg, x);
         }
         else {
             struct list_s *lst = f->objects_with_finalizers;
             list_set_item(lst, lst->count++, (uintptr_t)x);
+            list_set_item(lst, lst->count++, qindex);
         }
     }
-    list_free(marked);
+    LIST_FREE(marked);
 
     f->run_finalizers = run_finalizers;
 }
 
+
 static void deal_with_objects_with_finalizers(void)
 {
     /* for non-light finalizers */
@@ -436,11 +468,10 @@
 static void mark_visit_from_finalizer1(
     struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
-    if (f != NULL && f->run_finalizers != NULL) {
-        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
-                       ({
-                           mark_visit_possibly_overflow_object(item, pseg);
-                       }));
+    long i, count = list_count(f->run_finalizers);
+    for (i = 0; i < count; i += 2) {
+        object_t *x = (object_t *)list_item(f->run_finalizers, i);
+        mark_visit_possibly_overflow_object(x, pseg);
     }
 }
 
@@ -454,40 +485,6 @@
     mark_visit_from_finalizer1(get_priv_segment(0), &g_finalizers);
 }
 
-static void _execute_finalizers(struct finalizers_s *f)
-{
-    if (f->run_finalizers == NULL)
-        return;   /* nothing to do */
-
- restart:
-    if (f->running_next != NULL)
-        return;   /* in a nested invocation of execute_finalizers() */
-
-    uintptr_t next = 0, total = list_count(f->run_finalizers);
-    f->running_next = &next;
-
-    while (next < total) {
-        object_t *obj = (object_t *)list_item(f->run_finalizers, next);
-        list_set_item(f->run_finalizers, next, 0);
-        next++;
-
-        stmcb_finalizer(obj);
-    }
-    if (next == (uintptr_t)-1) {
-        /* transaction committed: the whole 'f' was freed */
-        return;
-    }
-    f->running_next = NULL;
-
-    if (f->run_finalizers->count > total) {
-        memmove(f->run_finalizers->items,
-                f->run_finalizers->items + total,
-                (f->run_finalizers->count - total) * sizeof(uintptr_t));
-        goto restart;
-    }
-
-    LIST_FREE(f->run_finalizers);
-}
 
 /* XXX: according to translator.backendopt.finalizer, getfield_gc
         for primitive types is a safe op in light finalizers.
@@ -495,43 +492,185 @@
         getfield on *dying obj*).
 */
 
+static void _trigger_finalizer_queues(struct finalizers_s *f)
+{
+    /* runs triggers of finalizer queues that have elements in the queue. May
+       NOT run outside of a transaction, but triggers never leave the
+       transactional zone.
+
+       returns true if there are also old-style finalizers to run */
+    assert(in_transaction(STM_PSEGMENT->pub.running_thread));
+
+    bool *to_trigger = (bool*)alloca(g_finalizer_triggers.count * sizeof(bool));
+    memset(to_trigger, 0, g_finalizer_triggers.count * sizeof(bool));
+
+    while (__sync_lock_test_and_set(&f->lock, 1) != 0) {
+        /* somebody is adding more finalizers (_commit_finalizer()) */
+        stm_spin_loop();
+    }
+
+    int count = list_count(f->run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(f->run_finalizers, i + 1);
+        dprintf(("qindex=%d\n", qindex));
+        to_trigger[qindex] = true;
+    }
+
+    __sync_lock_release(&f->lock);
+
+    // trigger now:
+    for (int i = 0; i < g_finalizer_triggers.count; i++) {
+        if (to_trigger[i]) {
+            dprintf(("invoke-finalizer-trigger(qindex=%d)\n", i));
+            g_finalizer_triggers.triggers[i]();
+        }
+    }
+}
+
+static bool _has_oldstyle_finalizers(struct finalizers_s *f)
+{
+    int count = list_count(f->run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(f->run_finalizers, i + 1);
+        if (qindex == -1)
+            return true;
+    }
+    return false;
+}
+
+static void _invoke_local_finalizers()
+{
+    /* called inside a transaction; invoke local triggers, process old-style
+     * local finalizers */
+    dprintf(("invoke_local_finalizers %lu\n", list_count(STM_PSEGMENT->finalizers->run_finalizers)));
+    if (list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)
+        && list_is_empty(g_finalizers.run_finalizers))
+        return;
+
+    struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
+    //try to run local triggers
+    if (STM_PSEGMENT->finalizers->running_trigger_now == NULL) {
+        // we are not recursively running them
+        STM_PSEGMENT->finalizers->running_trigger_now = pseg;
+        _trigger_finalizer_queues(STM_PSEGMENT->finalizers);
+        STM_PSEGMENT->finalizers->running_trigger_now = NULL;
+    }
+
+    // try to run global triggers
+    if (__sync_lock_test_and_set(&g_finalizers.running_trigger_now, pseg) == NULL) {
+        // nobody is already running these triggers (recursively)
+        _trigger_finalizer_queues(&g_finalizers);
+        g_finalizers.running_trigger_now = NULL;
+    }
+
+    if (!_has_oldstyle_finalizers(STM_PSEGMENT->finalizers))
+        return; // no oldstyle to run
+
+    object_t *obj;
+    while ((obj = stm_next_to_finalize(-1)) != NULL) {
+        stmcb_finalizer(obj);
+    }
+}
+
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
-    /* called between transactions */
+    /* called between transactions
+     * triggers not called here, since all should have been called already in _invoke_local_finalizers!
+     * run old-style finalizers (q_index=-1)
+     * queues that are not empty. */
+    dprintf(("invoke_general_finalizers %lu\n", list_count(g_finalizers.run_finalizers)));
+    if (list_is_empty(g_finalizers.run_finalizers))
+        return;
+
+    if (!_has_oldstyle_finalizers(&g_finalizers))
+        return; // no oldstyle to run
+
+    // run old-style finalizers:
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
     _stm_start_transaction(tl);
-    /* XXX: become inevitable, bc. otherwise, we would need to keep
-       around the original g_finalizers.run_finalizers to restore it
-       in case of an abort. */
-    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
-    /* did it work? */
-    if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {   /* no */
-        /* avoid blocking here, waiting for another INEV transaction.
-           If we did that, application code could not proceed (start the
-           next transaction) and it will not be obvious from the profile
-           why we were WAITing. */
-        _stm_commit_transaction();
-        stm_rewind_jmp_leaveframe(tl, &rjbuf);
-        return;
+
+    dprintf(("invoke_oldstyle_finalizers %lu\n", list_count(g_finalizers.run_finalizers)));
+    object_t *obj;
+    while ((obj = stm_next_to_finalize(-1)) != NULL) {
+        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        stmcb_finalizer(obj);
     }
 
+    _stm_commit_transaction();
+    stm_rewind_jmp_leaveframe(tl, &rjbuf);
+}
+
+object_t* stm_next_to_finalize(int queue_index) {
+    assert(STM_PSEGMENT->transaction_state != TS_NONE);
+
+    /* first check local run_finalizers queue, then global */
+    if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) {
+        struct list_s *lst = STM_PSEGMENT->finalizers->run_finalizers;
+        int count = list_count(lst);
+        for (int i = 0; i < count; i += 2) {
+            int qindex = (int)list_item(lst, i + 1);
+            if (qindex == queue_index) {
+                /* no need to become inevitable for local ones */
+                /* Remove obj from list and return it. */
+                object_t *obj = (object_t*)list_item(lst, i);
+                int remaining = count - i - 2;
+                if (remaining > 0) {
+                    memmove(&lst->items[i],
+                            &lst->items[i + 2],
+                            remaining * sizeof(uintptr_t));
+                }
+                lst->count -= 2;
+                return obj;
+            }
+        }
+    }
+
+    /* no local finalizers found, continue in global list */
+
     while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
         /* somebody is adding more finalizers (_commit_finalizer()) */
         stm_spin_loop();
     }
-    struct finalizers_s copy = g_finalizers;
-    assert(copy.running_next == NULL);
-    g_finalizers.run_finalizers = NULL;
+
+    struct list_s *lst = g_finalizers.run_finalizers;
+    int count = list_count(lst);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(lst, i + 1);
+        if (qindex == queue_index) {
+            /* XXX: become inevitable, bc. otherwise, we would need to keep
+               around the original g_finalizers.run_finalizers to restore it
+               in case of an abort. */
+            if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {
+                _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+                /* did it work? */
+                if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {   /* no */
+                    /* avoid blocking here, waiting for another INEV transaction.
+                       If we did that, application code could not proceed (start the
+                       next transaction) and it will not be obvious from the profile
+                       why we were WAITing. XXX: still true? */
+                    __sync_lock_release(&g_finalizers.lock);
+                    return NULL;
+                }
+            }
+
+            /* Remove obj from list and return it. */
+            object_t *obj = (object_t*)list_item(lst, i);
+            int remaining = count - i - 2;
+            if (remaining > 0) {
+                memmove(&lst->items[i],
+                        &lst->items[i + 2],
+                        remaining * sizeof(uintptr_t));
+            }
+            lst->count -= 2;
+
+            __sync_lock_release(&g_finalizers.lock);
+            return obj;
+        }
+    }
+
     /* others may add to g_finalizers again: */
     __sync_lock_release(&g_finalizers.lock);
 
-    if (copy.run_finalizers != NULL) {
-        _execute_finalizers(&copy);
-    }
-
-    _stm_commit_transaction();
-    stm_rewind_jmp_leaveframe(tl, &rjbuf);
-
-    LIST_FREE(copy.run_finalizers);
+    return NULL;
 }
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
--- a/c8/stm/finalizer.h
+++ b/c8/stm/finalizer.h
@@ -1,16 +1,20 @@
+#ifndef _STM_FINALIZER_H_
+#define _STM_FINALIZER_H_
+
+#include <stdint.h>
 
 /* see deal_with_objects_with_finalizers() for explanation of these fields */
 struct finalizers_s {
     long lock;
+    struct stm_priv_segment_info_s * running_trigger_now; /* our PSEG, if we are running triggers */
     struct list_s *objects_with_finalizers;
-    uintptr_t count_non_young;
+    struct list_s *probably_young_objects_with_finalizers; /* empty on g_finalizers! */
     struct list_s *run_finalizers;
-    uintptr_t *running_next;
 };
 
 static void mark_visit_from_finalizer_pending(void);
-static void deal_with_young_objects_with_finalizers(void);
-static void deal_with_old_objects_with_finalizers(void);
+static void deal_with_young_objects_with_destructors(void);
+static void deal_with_old_objects_with_destructors(void);
 static void deal_with_objects_with_finalizers(void);
 
 static void setup_finalizer(void);
@@ -27,19 +31,22 @@
 
 /* regular finalizers (objs from already-committed transactions) */
 static struct finalizers_s g_finalizers;
+static struct {
+    int count;
+    stm_finalizer_trigger_fn *triggers;
+} g_finalizer_triggers;
+
 
 static void _invoke_general_finalizers(stm_thread_local_t *tl);
+static void _invoke_local_finalizers(void);
 
 #define invoke_general_finalizers(tl)    do {   \
-    if (g_finalizers.run_finalizers != NULL)    \
-        _invoke_general_finalizers(tl);         \
+     _invoke_general_finalizers(tl);         \
 } while (0)
 
-static void _execute_finalizers(struct finalizers_s *f);
 
-#define any_local_finalizers() (STM_PSEGMENT->finalizers != NULL &&         \
-                               STM_PSEGMENT->finalizers->run_finalizers != NULL)
 #define exec_local_finalizers()  do {                   \
-    if (any_local_finalizers())                         \
-        _execute_finalizers(STM_PSEGMENT->finalizers);  \
+     _invoke_local_finalizers();                     \
 } while (0)
+
+#endif
diff --git a/c8/stm/fprintcolor.h b/c8/stm/fprintcolor.h
--- a/c8/stm/fprintcolor.h
+++ b/c8/stm/fprintcolor.h
@@ -1,3 +1,7 @@
+#ifndef _FPRINTCOLOR_H
+#define _FPRINTCOLOR_H
+
+
 /* ------------------------------------------------------------ */
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
@@ -40,3 +44,5 @@
 __attribute__((unused))
 static void stm_fatalerror(const char *format, ...)
      __attribute__((format (printf, 1, 2), noreturn));
+
+#endif
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -807,9 +807,9 @@
 
     LIST_FREE(marked_objects_to_trace);
 
-    /* weakrefs and execute old light finalizers */
+    /* weakrefs and execute old destructors */
     stm_visit_old_weakrefs();
-    deal_with_old_objects_with_finalizers();
+    deal_with_old_objects_with_destructors();
 
     /* cleanup */
     clean_up_segment_lists();
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
--- a/c8/stm/gcpage.h
+++ b/c8/stm/gcpage.h
@@ -1,3 +1,7 @@
+#ifndef _STM_GCPAGE_H_
+#define _STM_GCPAGE_H_
+
+#include <stdbool.h>
 
 /* Granularity when grabbing more unused pages: take 20 at a time */
 #define GCPAGE_NUM_PAGES   20
@@ -22,3 +26,9 @@
 static void major_collection_with_mutex(void);
 static bool largemalloc_keep_object_at(char *data);   /* for largemalloc.c */
 static bool smallmalloc_keep_object_at(char *data);   /* for smallmalloc.c */
+
+static inline bool mark_visited_test(object_t *obj);
+static bool is_overflow_obj_safe(struct stm_priv_segment_info_s *pseg, object_t *obj);
+static void mark_visit_possibly_overflow_object(object_t *obj, struct stm_priv_segment_info_s *pseg);
+
+#endif
diff --git a/c8/stm/list.h b/c8/stm/list.h
--- a/c8/stm/list.h
+++ b/c8/stm/list.h
@@ -1,5 +1,11 @@
+#ifndef _LIST_H
+#define _LIST_H
+
+
 #include <stdlib.h>
 #include <stdbool.h>
+#include <stdint.h>
+
 
 /************************************************************/
 
@@ -11,13 +17,13 @@
 
 static struct list_s *list_create(void) __attribute__((unused));
 
-static inline void list_free(struct list_s *lst)
+static inline void _list_free(struct list_s *lst)
 {
     free(lst);
 }
 
 #define LIST_CREATE(lst)  ((lst) = list_create())
-#define LIST_FREE(lst)  (list_free(lst), (lst) = NULL)
+#define LIST_FREE(lst)  (_list_free(lst), (lst) = NULL)
 
 
 static struct list_s *_list_grow(struct list_s *, uintptr_t);
@@ -245,3 +251,5 @@
     TREE_FIND(tree, addr, result, return false);
     return true;
 }
+
+#endif
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -3,6 +3,8 @@
 # include "core.h"  // silence flymake
 #endif
 
+#include "finalizer.h"
+
 /************************************************************/
 
 #define NURSERY_START         (FIRST_NURSERY_PAGE * 4096UL)
@@ -56,7 +58,6 @@
 
 /************************************************************/
 static object_t *find_existing_shadow(object_t *obj);
-#define GCWORD_MOVED  ((object_t *) -1)
 #define FLAG_SYNC_LARGE       0x01
 
 
@@ -441,23 +442,26 @@
     }
 }
 
-static void collect_objs_still_young_but_with_finalizers(void)
+static void collect_young_objects_with_finalizers(void)
 {
-    struct list_s *lst = STM_PSEGMENT->finalizers->objects_with_finalizers;
-    uintptr_t i, total = list_count(lst);
+    /* for real finalizers: in a minor collection, all young objs must survive! */
 
-    for (i = STM_PSEGMENT->finalizers->count_non_young; i < total; i++) {
+    struct list_s *lst = STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers;
+    long i, count = list_count(lst);
+    for (i = 0; i < count; i += 2) {
+        object_t *obj = (object_t *)list_item(lst, i);
+        uintptr_t qindex = list_item(lst, i + 1);
 
-        object_t *o = (object_t *)list_item(lst, i);
-        minor_trace_if_young(&o);
+        minor_trace_if_young(&obj);
 
-        /* was not actually movable */
-        assert(o == (object_t *)list_item(lst, i));
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, qindex);
     }
-    STM_PSEGMENT->finalizers->count_non_young = total;
+    list_clear(lst);
 }
 
 
+
 static void throw_away_nursery(struct stm_priv_segment_info_s *pseg)
 {
 #pragma push_macro("STM_PSEGMENT")
@@ -555,8 +559,7 @@
 
     collect_roots_in_nursery();
 
-    if (STM_PSEGMENT->finalizers != NULL)
-        collect_objs_still_young_but_with_finalizers();
+    collect_young_objects_with_finalizers();
 
     if (STM_PSEGMENT->active_queues != NULL)
         collect_active_queues();
@@ -568,7 +571,7 @@
     acquire_privatization_lock(STM_SEGMENT->segment_num);
     stm_move_young_weakrefs();
     release_privatization_lock(STM_SEGMENT->segment_num);
-    deal_with_young_objects_with_finalizers();
+    deal_with_young_objects_with_destructors();
 
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
 
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -1,3 +1,8 @@
+#ifndef _STM_NURSERY_H_
+#define _STM_NURSERY_H_
+
+#include <stdint.h>
+#include <stdbool.h>
 
 #define NSE_SIGPAUSE   _STM_NSE_SIGNAL_MAX
 #define NSE_SIGABORT   _STM_NSE_SIGNAL_ABORT
@@ -44,3 +49,11 @@
 
 #define must_abort()   is_abort(STM_SEGMENT->nursery_end)
 static object_t *find_shadow(object_t *obj);
+
+
+#define GCWORD_MOVED  ((object_t *) -1)
+static inline bool _is_young(object_t *obj);
+static inline struct object_s *mark_loc(object_t *obj);
+static inline bool _is_from_same_transaction(object_t *obj);
+
+#endif
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -5,6 +5,8 @@
 
 #include <signal.h>
 #include <fcntl.h>           /* For O_* constants */
+#include "signal_handler.h"
+#include "fprintcolor.h"
 
 static void setup_mmap(char *reason)
 {
@@ -100,8 +102,8 @@
         pr->nursery_objects_shadows = tree_create();
         pr->callbacks_on_commit_and_abort[0] = tree_create();
         pr->callbacks_on_commit_and_abort[1] = tree_create();
-        pr->young_objects_with_light_finalizers = list_create();
-        pr->old_objects_with_light_finalizers = list_create();
+        pr->young_objects_with_destructors = list_create();
+        pr->old_objects_with_destructors = list_create();
 
         pr->last_commit_log_entry = &commit_log_root;
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
@@ -145,19 +147,19 @@
     for (i = 0; i < NB_SEGMENTS; i++) {
         struct stm_priv_segment_info_s *pr = get_priv_segment(i);
         assert(list_is_empty(pr->objects_pointing_to_nursery));
-        list_free(pr->objects_pointing_to_nursery);
-        list_free(pr->old_objects_with_cards_set);
-        list_free(pr->modified_old_objects);
+        LIST_FREE(pr->objects_pointing_to_nursery);
+        LIST_FREE(pr->old_objects_with_cards_set);
+        LIST_FREE(pr->modified_old_objects);
         assert(list_is_empty(pr->large_overflow_objects));
-        list_free(pr->large_overflow_objects);
-        list_free(pr->young_weakrefs);
-        list_free(pr->old_weakrefs);
+        LIST_FREE(pr->large_overflow_objects);
+        LIST_FREE(pr->young_weakrefs);
+        LIST_FREE(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);
         tree_free(pr->nursery_objects_shadows);
         tree_free(pr->callbacks_on_commit_and_abort[0]);
         tree_free(pr->callbacks_on_commit_and_abort[1]);
-        list_free(pr->young_objects_with_light_finalizers);
-        list_free(pr->old_objects_with_light_finalizers);
+        LIST_FREE(pr->young_objects_with_destructors);
+        LIST_FREE(pr->old_objects_with_destructors);
         if (pr->active_queues) tree_free(pr->active_queues);
     }
 
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -710,13 +710,13 @@
 void stm_force_transaction_break(stm_thread_local_t *tl);
 
 
-/* Support for light finalizers.  This is a simple version of
+/* Support for destructors.  This is a simple version of
    finalizers that guarantees not to do anything fancy, like not
    resurrecting objects. */
-extern void (*stmcb_light_finalizer)(object_t *);
-void stm_enable_light_finalizer(object_t *);
+extern void (*stmcb_destructor)(object_t *);
+void stm_enable_destructor(object_t *);
 
-/* Support for regular finalizers.  Unreachable objects with
+/* XXX: Support for regular finalizers.  Unreachable objects with
    finalizers are kept alive, as well as everything they point to, and
    stmcb_finalizer() is called after the major GC.  If there are
    several objects with finalizers that reference each other in a
@@ -730,8 +730,14 @@
    transaction.  For older objects, the finalizer is called from a
    random thread between regular transactions, in a new custom
    transaction. */
-extern void (*stmcb_finalizer)(object_t *);
-object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
+typedef void (*stm_finalizer_trigger_fn)(void);
+void (*stmcb_finalizer)(object_t *);
+void stm_setup_finalizer_queues(int number, stm_finalizer_trigger_fn *triggers);
+void stm_enable_finalizer(int queue_index, object_t *obj);
+
+/* Returns the next object that supposedly died and should have its finalizer
+   called. XXX: This function turns the transaction inevitable. */
+object_t *stm_next_to_finalize(int queue_index);
 
 
 /* dummies for now: */
@@ -742,7 +748,7 @@
    'object_t *'.  Note that the type 'stm_hashtable_t' is not an
    object type at all; you need to allocate and free it explicitly.
    If you want to embed the hashtable inside an 'object_t' you
-   probably need a light finalizer to do the freeing. */
+   probably need a destructor to do the freeing. */
 typedef struct stm_hashtable_s stm_hashtable_t;
 typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
 
@@ -797,8 +803,8 @@
 /* Queues.  The items you put() and get() back are in random order.
    Like hashtables, the type 'stm_queue_t' is not an object type at
    all; you need to allocate and free it explicitly.  If you want to
-   embed the queue inside an 'object_t' you probably need a light
-   finalizer to do the freeing. */
+   embed the queue inside an 'object_t' you probably need a destructor
+   to do the freeing. */
 typedef struct stm_queue_s stm_queue_t;
 
 stm_queue_t *stm_queue_create(void);
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -52,7 +52,6 @@
 /*void stm_write(object_t *obj); use _checked_stm_write() instead */
 object_t *stm_allocate(ssize_t size_rounded_up);
 object_t *stm_allocate_weakref(ssize_t size_rounded_up);
-object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
 object_t *stm_allocate_noconflict(ssize_t size_rounded_up);
 
 /*void stm_write_card(); use _checked_stm_write_card() instead */
@@ -206,11 +205,14 @@
 bool (*_stm_smallmalloc_keep)(char *data);
 void _stm_smallmalloc_sweep_test(void);
 
+void (*stmcb_destructor)(object_t *);
+void stm_enable_destructor(object_t *);
 
-void (*stmcb_light_finalizer)(object_t *);
-void stm_enable_light_finalizer(object_t *);
-
+typedef void (*stm_finalizer_trigger_fn)(void);
 void (*stmcb_finalizer)(object_t *);
+void stm_setup_finalizer_queues(int number, stm_finalizer_trigger_fn *triggers);
+void stm_enable_finalizer(int queue_index, object_t *obj);
+object_t *stm_next_to_finalize(int queue_index);
 
 typedef struct stm_hashtable_s stm_hashtable_t;
 typedef ... stm_hashtable_entry_t;
@@ -749,15 +751,19 @@
     return o
 
 def stm_allocate_with_finalizer(size):
-    o = lib.stm_allocate_with_finalizer(size)
+    # OLD-Style finalizers!
+    o = lib.stm_allocate(size)
     tid = 42 + size
     lib._set_type_id(o, tid)
+    lib.stm_enable_finalizer(-1, o)
     return o
 
 def stm_allocate_with_finalizer_refs(n):
-    o = lib.stm_allocate_with_finalizer(HDR + n * WORD)
+    # OLD-Style finalizers!
+    o = lib.stm_allocate(HDR + n * WORD)
     tid = 421420 + n
     lib._set_type_id(o, tid)
+    lib.stm_enable_finalizer(-1, o)
     return o
 
 SIZEOF_HASHTABLE_OBJ = 16 + lib.SIZEOF_MYOBJ
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -2,13 +2,22 @@
 import py
 
 
-class TestLightFinalizer(BaseTest):
+class TestDestructors(BaseTest):
 
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
         #
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger():
+            # triggers not needed for destructors
+            assert False
+        self._trigger_keepalive = trigger
+        triggers = ffi.new("stm_finalizer_trigger_fn*", trigger)
+        lib.stm_setup_finalizer_queues(1, triggers)
+        triggers = None
+        #
         @ffi.callback("void(object_t *)")
-        def light_finalizer(obj):
+        def destructor(obj):
             assert stm_get_obj_size(obj) == 48
             segnum = lib.current_segment_num()
             tlnum = '?'
@@ -17,21 +26,21 @@
                     if tl.last_associated_segment_num == segnum:
                         tlnum = n
                         break
-            self.light_finalizers_called.append((obj, tlnum))
-        self.light_finalizers_called = []
-        lib.stmcb_light_finalizer = light_finalizer
-        self._light_finalizer_keepalive = light_finalizer
+            self.destructors_called.append((obj, tlnum))
+        self.destructors_called = []
+        lib.stmcb_destructor = destructor
+        self._destructor_keepalive = destructor
 
     def teardown_method(self, meth):
-        lib.stmcb_light_finalizer = ffi.NULL
+        lib.stmcb_destructor = ffi.NULL
         BaseTest.teardown_method(self, meth)
 
     def expect_finalized(self, objs, from_tlnum=None):
-        assert [obj for (obj, tlnum) in self.light_finalizers_called] == objs
+        assert [obj for (obj, tlnum) in self.destructors_called] == objs
         if from_tlnum is not None:
-            for obj, tlnum in self.light_finalizers_called:
+            for obj, tlnum in self.destructors_called:
                 assert tlnum == from_tlnum
-        self.light_finalizers_called = []
+        self.destructors_called = []
 
     def test_no_finalizer(self):
         self.start_transaction()
@@ -39,45 +48,45 @@
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_young_light_finalizer(self):
+    def test_young_destructor(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.expect_finalized([])
         self.commit_transaction()
         self.expect_finalized([lp1], from_tlnum=0)
 
-    def test_young_light_finalizer_survives(self):
+    def test_young_destructor_survives(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)       # stays alive
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_young_light_finalizer_aborts(self):
+    def test_young_destructor_aborts(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.expect_finalized([])
         self.abort_transaction()
         self.start_transaction()
         self.expect_finalized([lp1], from_tlnum=0)
 
-    def test_old_light_finalizer(self):
+    def test_old_destructor(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_old_light_finalizer_2(self):
+    def test_old_destructor_2(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
@@ -86,10 +95,10 @@
         self.expect_finalized([lp1])
         self.commit_transaction()
 
-    def test_old_light_finalizer_survives(self):
+    def test_old_destructor_survives(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
@@ -98,13 +107,13 @@
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_old_light_finalizer_segment(self):
+    def test_old_destructor_segment(self):
         self.start_transaction()
         #
         self.switch(1)
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
@@ -114,10 +123,10 @@
         stm_major_collect()
         self.expect_finalized([lp1], from_tlnum=1)
 
-    def test_old_light_finalizer_aborts(self):
+    def test_old_destructor_aborts(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         self.commit_transaction()
         #
@@ -126,10 +135,10 @@
         self.abort_transaction()
         self.expect_finalized([])
 
-    def test_overflow_light_finalizer_aborts(self):
+    def test_overflow_destructor_aborts(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
@@ -139,14 +148,212 @@
         self.expect_finalized([lp1], from_tlnum=0)
 
 
+class TestRegularFinalizerQueues(BaseTest):
+    expect_content_character = None
+    run_major_collect_in_trigger = False
 
-class TestRegularFinalizer(BaseTest):
+    def setup_method(self, meth):
+        BaseTest.setup_method(self, meth)
+        #
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger0():
+            self.queues_triggered.append(0)
+            if self.run_major_collect_in_trigger:
+                stm_major_collect()
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger1():
+            self.queues_triggered.append(1)
+            if self.run_major_collect_in_trigger:
+                stm_major_collect()
+        self._trigger_keepalive = [trigger0, trigger1]
+        triggers = ffi.new("stm_finalizer_trigger_fn[]", self._trigger_keepalive)
+        lib.stm_setup_finalizer_queues(2, triggers)
+        #
+        self.queues_triggered = []
+
+    def get_queues_triggered(self):
+        res = set(self.queues_triggered)
+        self.queues_triggered = []
+        return res
+
+    def get_to_finalize(self, qindex):
+        obj = lib.stm_next_to_finalize(qindex)
+        res = []
+        while obj:
+            res.append(obj)
+            obj = lib.stm_next_to_finalize(qindex)
+        return res
+
+
+
+    def test_no_finalizer(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        stm_major_collect()
+        assert not self.get_queues_triggered()
+
+    def test_no_finalizer_in_minor_collection(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_finalizer(0, lp1)
+        stm_minor_collect()
+        assert not self.get_queues_triggered()
+
+    def test_finalizer_in_major_collection(self):
+        self.start_transaction()
+        for repeat in range(4):
+            lp1 = stm_allocate(48)
+            lp2 = stm_allocate(48)
+            lp3 = stm_allocate(48)
+            if repeat % 2 == 0: # register on young
+                lib.stm_enable_finalizer(0, lp1)
+                lib.stm_enable_finalizer(1, lp2)
+                lib.stm_enable_finalizer(1, lp3)
+            assert not self.get_queues_triggered()
+            self.push_roots([lp1, lp2, lp3])
+            stm_minor_collect()
+            lp1, lp2, lp3 = self.pop_roots()
+            if repeat % 2 == 1: # register on old
+                lib.stm_enable_finalizer(0, lp1)
+                lib.stm_enable_finalizer(1, lp2)
+                lib.stm_enable_finalizer(1, lp3)
+            print repeat, lp1, lp2, lp3
+            stm_major_collect()
+            assert {0, 1} == self.get_queues_triggered()
+            assert [lp1] == self.get_to_finalize(0)
+            assert [lp2, lp3] == self.get_to_finalize(1)
+
+    def test_finalizer_from_other_thread(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_finalizer(0, lp1)
+        stm_set_char(lp1, 'H')
+        self.expect_content_character = 'H'
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        print lp1
+        #
+        self.switch(1)
+        self.start_transaction()
+        stm_major_collect()
+        assert not self.get_queues_triggered()
+        #
+        self.switch(0)
+        assert {0} == self.get_queues_triggered()
+        assert [lp1] == self.get_to_finalize(0)
+
+    def test_finalizer_ordering(self):
+        self.start_transaction()
+        lp1 = stm_allocate_refs(1)
+        lp2 = stm_allocate_refs(1)
+        lp3 = stm_allocate_refs(1)
+        lib.stm_enable_finalizer(1, lp1)
+        lib.stm_enable_finalizer(1, lp2)
+        lib.stm_enable_finalizer(1, lp3)
+
+        print lp1, lp2, lp3
+        stm_set_ref(lp3, 0, lp1)
+        stm_set_ref(lp1, 0, lp2)
+
+        self.push_roots([lp1, lp2, lp3])
+        stm_minor_collect()
+        lp1, lp2, lp3 = self.pop_roots()
+
+        stm_major_collect()
+        assert {1} == self.get_queues_triggered()
+        assert [lp3] == self.get_to_finalize(1)
+
+    def test_finalizer_extra_transaction(self):
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        lib.stm_enable_finalizer(0, lp1)
+        self.push_root(lp1)
+        self.commit_transaction()
+
+        self.start_transaction()
+        lp1b = self.pop_root()
+        # assert lp1b == lp1 <- lp1 can be in nursery now
+        assert not self.get_queues_triggered()
+        self.commit_transaction() # finalizer-obj moved to global queue
+        assert not self.get_queues_triggered()
+
+        self.start_transaction()
+        stm_major_collect()
+        assert {0} == self.get_queues_triggered()
+        self.commit_transaction() # invoke finalizers (triggers again...)
+        self.start_transaction()
+        assert {0} == self.get_queues_triggered()
+        assert [lp1b] == self.get_to_finalize(0)
+
+    def test_run_cb_for_all_threads(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_finalizer(0, lp1)
+        print lp1
+        #
+        self.switch(1)
+        self.start_transaction()
+        lp2 = stm_allocate(56)
+        lib.stm_enable_finalizer(1, lp2)
+        print lp2
+
+        assert not self.get_queues_triggered()
+        stm_major_collect()
+        assert len(self.get_to_finalize(1)) == 1
+        self.switch(0)
+        assert self.get_queues_triggered() == {0, 1}
+        assert len(self.get_to_finalize(0)) == 1
+
+    def test_run_major_collect_in_trigger(self):
+        self.run_major_collect_in_trigger  = True
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        lp2 = stm_allocate(32)
+        lp3 = stm_allocate(32)
+        lib.stm_enable_finalizer(0, lp1)
+        lib.stm_enable_finalizer(1, lp2)
+        lib.stm_enable_finalizer(1, lp3)
+        print lp1, lp2, lp3
+        stm_major_collect()
+        assert 1 == len(self.get_to_finalize(0))
+        assert 2 == len(self.get_to_finalize(1))
+
+    def test_new_objects_w_finalizers(self):
+        self.switch(2)
+        self.start_transaction()
+        lp1 = stm_allocate_refs(3)
+        lp2 = stm_allocate_refs(3)
+        lib.stm_enable_finalizer(0, lp1)
+        lib.stm_enable_finalizer(0, lp2)
+        stm_set_ref(lp1, 0, lp2)
+
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        lp2 = stm_get_ref(lp1, 0)
+        # lp1, lp2 have WB_EXECUTED objs
+
+        assert not self.get_queues_triggered()
+        stm_major_collect()
+        assert [lp1] == self.get_to_finalize(0)
+
+
+class TestOldStyleRegularFinalizer(BaseTest):
     expect_content_character = None
     run_major_collect_in_finalizer = False
 
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
         #
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger():
+            # triggers not needed for oldstyle-finalizer tests
+            assert False
+        self._trigger_keepalive = trigger
+        triggers = ffi.new("stm_finalizer_trigger_fn*", trigger)
+        lib.stm_setup_finalizer_queues(1, triggers)
+        #
         @ffi.callback("void(object_t *)")
         def finalizer(obj):
             print "finalizing!", obj
@@ -161,7 +368,12 @@
         self._finalizer_keepalive = finalizer
 
     def expect_finalized(self, objs):
-        assert self.finalizers_called == objs
+        if isinstance(objs, int):
+            assert len(self.finalizers_called) == objs
+        else:
+            print objs
+            print self.finalizers_called
+            assert self.finalizers_called == objs
         self.finalizers_called = []
 
     def test_no_finalizer(self):
@@ -182,8 +394,11 @@
             lp1 = stm_allocate_with_finalizer(48)
             lp2 = stm_allocate_with_finalizer(48)
             lp3 = stm_allocate_with_finalizer(48)
+            self.expect_finalized([])
+            self.push_roots([lp1, lp2, lp3])
+            stm_minor_collect()
+            lp1, lp2, lp3 = self.pop_roots()
             print repeat, lp1, lp2, lp3
-            self.expect_finalized([])
             stm_major_collect()
             self.expect_finalized([lp1, lp2, lp3])
 
@@ -192,6 +407,9 @@
         lp1 = stm_allocate_with_finalizer(48)
         stm_set_char(lp1, 'H')
         self.expect_content_character = 'H'
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
         print lp1
         #
         self.switch(1)
@@ -210,6 +428,11 @@
         print lp1, lp2, lp3
         stm_set_ref(lp3, 0, lp1)
         stm_set_ref(lp1, 0, lp2)
+
+        self.push_roots([lp1, lp2, lp3])
+        stm_minor_collect()
+        lp1, lp2, lp3 = self.pop_roots()
+
         stm_major_collect()
         self.expect_finalized([lp3])
 
@@ -222,16 +445,16 @@
 
         self.start_transaction()
         lp1b = self.pop_root()
-        assert lp1b == lp1
+        # assert lp1b == lp1 <- lp1 can be in nursery now
         self.expect_finalized([])
-        self.commit_transaction()
+        self.commit_transaction() # finalizer-obj moved to global queue
         self.expect_finalized([])
 
         self.start_transaction()
         stm_major_collect()
         self.expect_finalized([])
-        self.commit_transaction()
-        self.expect_finalized([lp1])
+        self.commit_transaction() # invoke finalizers
+        self.expect_finalized([lp1b])
 
     def test_run_cb_for_all_threads(self):
         self.start_transaction()
@@ -246,7 +469,7 @@
         self.expect_finalized([])
         stm_major_collect()
         self.switch(0)
-        self.expect_finalized([lp2, lp1])
+        self.expect_finalized(2)
 
     def test_run_major_collect_in_finalizer(self):
         self.run_major_collect_in_finalizer = True
diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py
--- a/c8/test/test_hashtable.py
+++ b/c8/test/test_hashtable.py
@@ -43,8 +43,8 @@
         BaseTest.setup_method(self, meth)
         #
         @ffi.callback("void(object_t *)")
-        def light_finalizer(obj):
-            print 'light_finalizer:', obj
+        def destructor(obj):
+            print 'destructor:', obj
             try:
                 assert lib._get_type_id(obj) == 421419
                 self.seen_hashtables -= 1
@@ -54,20 +54,20 @@
                 self.errors.append(sys.exc_info()[2])
                 raise
 
-        lib.stmcb_light_finalizer = light_finalizer
-        self._light_finalizer_keepalive = light_finalizer
+        lib.stmcb_destructor = destructor
+        self._destructor_keepalive = destructor
         self.seen_hashtables = 0
         self.errors = []
 
     def teardown_method(self, meth):
         BaseTest.teardown_method(self, meth)
-        lib.stmcb_light_finalizer = ffi.NULL
+        lib.stmcb_destructor = ffi.NULL
         assert self.errors == []
         assert self.seen_hashtables == 0
 
     def allocate_hashtable(self):
         h = stm_allocate_hashtable()
-        lib.stm_enable_light_finalizer(h)
+        lib.stm_enable_destructor(h)
         self.seen_hashtables += 1
         return h
 
diff --git a/c8/test/test_list.py b/c8/test/test_list.py
--- a/c8/test/test_list.py
+++ b/c8/test/test_list.py
@@ -6,7 +6,7 @@
 ffi = cffi.FFI()
 ffi.cdef("""
 struct list_s *list_create(void);
-void list_free(struct list_s *lst);
+void _list_free(struct list_s *lst);
 struct list_s *list_append(struct list_s *lst, uintptr_t item);
 uintptr_t list_count(struct list_s *lst);
 uintptr_t list_item(struct list_s *lst, uintptr_t index);
@@ -144,5 +144,5 @@
     assert lib.list_count(a) == 17 + 7
     for i, expected in enumerate(range(103, 120) + range(113, 120)):
         assert lib.list_item(a, i) == expected
-    lib.list_free(b)
-    lib.list_free(a)
+    lib._list_free(b)
+    lib._list_free(a)
diff --git a/c8/test/test_queue.py b/c8/test/test_queue.py
--- a/c8/test/test_queue.py
+++ b/c8/test/test_queue.py
@@ -13,8 +13,8 @@
         BaseTest.setup_method(self, meth)
         #
         @ffi.callback("void(object_t *)")
-        def light_finalizer(obj):
-            print 'light_finalizer:', obj
+        def destructor(obj):
+            print 'destructor:', obj
             try:
                 assert lib._get_type_id(obj) == 421417
                 self.seen_queues -= 1
@@ -24,20 +24,20 @@
                 self.errors.append(sys.exc_info()[2])
                 raise
 
-        lib.stmcb_light_finalizer = light_finalizer
-        self._light_finalizer_keepalive = light_finalizer
+        lib.stmcb_destructor = destructor
+        self._destructor_keepalive = destructor
         self.seen_queues = 0
         self.errors = []
 
     def teardown_method(self, meth):
         BaseTest.teardown_method(self, meth)
-        lib.stmcb_light_finalizer = ffi.NULL
+        lib.stmcb_destructor = ffi.NULL
         assert self.errors == []
         assert self.seen_queues == 0
 
     def allocate_queue(self):
         q = stm_allocate_queue()
-        lib.stm_enable_light_finalizer(q)
+        lib.stm_enable_destructor(q)
         self.seen_queues += 1
         return q
 
diff --git a/c8/test/test_zrandom.py b/c8/test/test_zrandom.py
--- a/c8/test/test_zrandom.py
+++ b/c8/test/test_zrandom.py
@@ -411,13 +411,20 @@
     global_state.add_noconfl_obj(r)
 
 def op_allocate_ref(ex, global_state, thread_state):
-    num = str(global_state.rnd.randrange(1, 1000))
+    g_rnd = global_state.rnd
+    num = str(g_rnd.randrange(1, 1000))
     r = global_state.get_new_root_name(True, num)
     thread_state.push_roots(ex)
     ex.do('%s = stm_allocate_refs(%s)' % (r, num))
     ex.do('# 0x%x' % (int(ffi.cast("uintptr_t", ex.content[r]))))
     thread_state.transaction_state.add_root(r, "ffi.NULL", True)
 
+    if g_rnd.randrange(10) == 0:
+        # XXX: should empty the queues sometimes! but calling next_to_finalize
+        # *may* turn inevitable (but only if there is something in the queue).
+        # so that's a bit involved...
+        ex.do('lib.stm_enable_finalizer(%s, %s)' % (g_rnd.randrange(2), r))
+
     thread_state.pop_roots(ex)
     thread_state.reload_roots(ex)
     thread_state.register_root(r)
@@ -585,6 +592,25 @@
         N_THREADS = self.NB_THREADS
         ex = Exec(self)
         ex.do("################################################################\n"*10)
+        ex.do("""
+
+ at ffi.callback("stm_finalizer_trigger_fn")
+def trigger0():
+    pass
+ at ffi.callback("stm_finalizer_trigger_fn")
+def trigger1():
+    pass
+self._trigger_keepalive = [trigger0, trigger1]
+triggers = ffi.new("stm_finalizer_trigger_fn[]", self._trigger_keepalive)
+lib.stm_setup_finalizer_queues(2, triggers)
+#
+ at ffi.callback("void(object_t *)")
+def finalizer(obj):
+    pass
+lib.stmcb_finalizer = finalizer
+self._finalizer_keepalive = finalizer
+        """)
+
         ex.do('# initialization')
 
         global_state = GlobalState(ex, rnd)
@@ -624,7 +650,7 @@
         ]
         possible_actions = [item for sublist in possible_actions for item in sublist]
         print possible_actions
-        for _ in range(2000):
+        for _ in range(3000):
             # make sure we are in a transaction:
             curr_thread = op_switch_thread(ex, global_state, curr_thread)
 

From pypy.commits at gmail.com  Thu Feb  9 03:57:25 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 00:57:25 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: add comment for
 vmprof_execute_code behaviour, extend test which now passes
Message-ID: <589c2ef5.ce9adf0a.f9907.d581@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90015:6a9a8962927b
Date: 2017-02-09 09:56 +0100
http://bitbucket.org/pypy/pypy/changeset/6a9a8962927b/

Log:	add comment for vmprof_execute_code behaviour, extend test which now
	passes

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -204,11 +204,21 @@
             unique_id = get_code_fn(*args)._vmprof_unique_id
             unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
+            #
+            # Signals can occur at the two places (1) and (2), that will
+            # have added a stack entry, but the function __vmprof_eval_vmprof
+            # is not entered. This behaviour will swallow one Python stack frame
+            #
+            # Current fix: vmprof will discard this sample. those happen
+            # very infrequent
+            #
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
+                # (1) signal here
                 try:
                     return func(*args)
                 finally:
+                    # (2) signal here
                     leave_code(x)
             else:
                 return decorated_jitted_function(unique_id, *args)
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -347,7 +347,7 @@
     ssize_t count;
     int version;
     int flags;
-    int memory, lines, native;
+    int memory = 0, lines = 0, native = 0;
     orig_pos = lseek(fileno, 0, SEEK_CUR);
 
     lseek(fileno, 5*WORD_SIZE, SEEK_SET);
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -138,7 +138,7 @@
 #endif
 
 int vmp_walk_and_record_stack(PY_STACK_FRAME_T *frame, void ** result,
-                              int max_depth, int native_skip, intptr_t pc) {
+                              int max_depth, intptr_t pc) {
 
     // called in signal handler
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
@@ -155,15 +155,17 @@
     int ret = unw_init_local(&cursor, &uc);
     if (ret < 0) {
         // could not initialize lib unwind cursor and context
-        return -1;
+        return 0;
     }
 
-    while (native_skip > 0) {
+    while (1) {
+        if (unw_is_signal_frame(&cursor)) {
+            break;
+        }
         int err = unw_step(&cursor);
         if (err <= 0) {
             return 0;
         }
-        native_skip--;
     }
 
     //printf("stack trace:\n");
@@ -209,31 +211,39 @@
 #else
             {
 #endif
-                if (top_most_frame == NULL) {
-                    break;
+                if (top_most_frame != NULL) {
+                    top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth);
+                } else {
+                    // Signals can occur at the two places (1) and (2), that will
+                    // have added a stack entry, but the function __vmprof_eval_vmprof
+                    // is not entered. This behaviour will swallow one Python stack frames
+                    //
+                    // (1) PyPy: enter_code happened, but __vmprof_eval_vmprof was not called
+                    // (2) PyPy: __vmprof_eval_vmprof was returned, but exit_code was not called
+                    //
+                    // destroy this sample, as it would display a strage sample
+                    return 0;
                 }
-                top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth);
             }
         } else if (vmp_ignore_ip((intptr_t)func_addr)) {
-            // this is an instruction pointer that should be ignored,
+            // this is an instruction pointer that should be ignored
             // (that is any function name in the mapping range of
-            //  cpython, but of course not extenstions in site-packages))
-            //printf("ignoring %s\n", info.dli_sname);
+            //  cpython or libpypy-c.so, but of course not
+            //  extenstions in site-packages)
         } else {
             // mark native routines with the first bit set,
             // this is possible because compiler align to 8 bytes.
             //
-
 #ifdef PYPY_JIT_CODEMAP
             if (func_addr == 0 && top_most_frame->kind == VMPROF_JITTED_TAG) {
                 intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
-                n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
+                depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
                 frame = FRAME_STEP(frame);
-            } else if (func_addr != 0) {
+            } else if (func_addr != 0x0) {
                 depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
             }
 #else
-            if (func_addr != 0) {
+            if (func_addr != 0x0) {
                 depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
             }
 #endif
@@ -241,8 +251,10 @@
 
         int err = unw_step(&cursor);
         if (err == 0) {
+            //printf("sample ended\n");
             break;
         } else if (err < 0) {
+            //printf("sample is broken\n");
             return 0; // this sample is broken, cannot walk it fully
         }
     }
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.h b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.h
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
@@ -3,7 +3,7 @@
 #include "vmprof.h"
 
 int vmp_walk_and_record_stack(PY_STACK_FRAME_T * frame, void **data,
-                              int max_depth, int native_skip, intptr_t pc);
+                              int max_depth, intptr_t pc);
 
 int vmp_native_enabled(void);
 int vmp_native_enable(void);
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -101,15 +101,7 @@
     }
     frame = current->frame;
 #endif
-    // skip over
-    // _sigtramp
-    // sigprof_handler
-    // vmp_walk_and_record_stack
-#ifdef __unix__
-    return vmp_walk_and_record_stack(frame, result, max_depth, 4, pc);
-#else
-    return vmp_walk_and_record_stack(frame, result, max_depth, 3, pc);
-#endif
+    return vmp_walk_and_record_stack(frame, result, max_depth, pc);
 }
 
 /* *************************************************************
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -151,7 +151,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.lltypesystem import rffi, lltype
 def test_native():
-    eci = ExternalCompilationInfo(compile_extra=['-g','-O2'],
+    eci = ExternalCompilationInfo(compile_extra=['-g','-O1'],
             separate_module_sources=["""
             RPY_EXTERN int native_func(void) {
                 int j = 0;
@@ -209,10 +209,24 @@
         tree = prof.get_tree()
         p = PrettyPrinter()
         p._print_tree(tree)
-        assert tree.name.startswith("n:pypy_g_main")
+        def walk(tree, symbols):
+            symbols.append(tree.name)
+            if len(tree.children) == 0:
+                return
+            assert len(tree.children) == 1
+            for child in tree.children.values():
+                walk(child, symbols)
+        symbols = []
+        walk(tree, symbols)
+        not_found = ['n:pypy_g_main', 'n:native_func', 'n:pypy_g_f',
+            'n:pypy_g_main']
+        for sym in symbols:
+            for i,name in enumerate(not_found):
+                if sym.startswith(name):
+                    del not_found[i]
+                    break
+        assert not_found == []
 
-    #assert f() == 0
-    #assert os.path.exists(tmpfilename)
     fn = compile(f, [], gcpolicy="minimark")
     assert fn() == 0
     try:

From pypy.commits at gmail.com  Thu Feb  9 04:57:46 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 09 Feb 2017 01:57:46 -0800 (PST)
Subject: [pypy-commit] stmgc default: improve readme
Message-ID: <589c3d1a.54b61c0a.ed992.939e@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2006:dc6f36e4de5b
Date: 2017-02-09 10:57 +0100
http://bitbucket.org/pypy/stmgc/changeset/dc6f36e4de5b/

Log:	improve readme

diff --git a/README.txt b/README.md
rename from README.txt
rename to README.md
--- a/README.txt
+++ b/README.md
@@ -1,28 +1,158 @@
 
-STM-GC
-======
+# STM-GC
 
 Welcome!
 
 This is a C library that combines a GC with STM capabilities.
 It is meant to be a general library that can be used in C programs.
 
-The library interface is in "c4/stmgc.h".
+The library interface is in `c8/stmgc.h`.
 
 Progress (these revisions are roughly stable versions, pick the last one):
-- 3aea86a96daf: last rev of "c3", the previous version
-- f1ccf5bbcb6f: first step, working but with no GC
-- 8da924453f29: minor collection seems to be working, no major GC
-- e7249873dcda: major collection seems to be working
+ - 3af462f
 
-The file "c4/doc-objects.txt" contains some low-level explanations.
+Run tests with `py.test`.
 
-Run tests with "py.test".
-
-A demo program can be found in "c4/demo1.c".
-It can be built with "make debug-demo1" or "make build-demo1".
+Demo programs can be found in `c8/demo/`.
 
 The plan is to use this C code directly with PyPy, and not write
 manually the many calls to the shadow stack and the barrier functions.
 But the manual way is possible too, say when writing a small interpreter
 directly in C.
+
+
+# Other resources
+
+http://doc.pypy.org/en/latest/stm.html
+
+# How to run things
+
+## Get PyPy 
+
+ 1. `hg clone https://bitbucket.org/pypy/pypy` (this will take a while, but you
+    can continue with the instructions below)
+ 2. `hg checkout stmgc-c8`
+
+
+## Get STMGC
+
+ 1. `hg clone https://bitbucket.org/pypy/stmgc`
+ 2. `gcc-seg-gs/README.txt` mentions which GCC version should work. Maybe -fXXX
+    flags mentioned at the end are still needed for compiling full PyPy-STM
+
+The folder `c8` contains the current version of the STMGC library
+
+### Project layout of c8
+
+ - `stmgc.h`: the main header file for the library
+ - `stm/`: 
+    - For the GC part: 
+       + `nursery`: minor collection
+       + `gcpage`: major collection
+       + `largemalloc`, `smallmalloc`: object allocation
+       + `finalizer`: object finalizer support
+       + `weakref`: weak references support
+    - For the STM part:
+       + `core`: commit, abort, barrier logic of STM
+       + `sync`: segment management and thread support
+       + `pages`: management of page metadata
+       + `signal_handler`: manages pages together with `pages`
+       + `locks`: a set of locks to protect segments
+       + `rewind_setjmp`: setjmp/longjmp implementation that supports arbitrary rollback
+       + `forksupport`: support for forking an STM process
+       + `extra`: on-commit and on-abort callback mechanism
+       + `detach`: transaction detach mechanism (optimised transactional zones)
+    - Misc:
+       + `fprintcolor`: colourful debug output
+       + `hash_id`: PyPy-compatible identity and identity-hash functionality
+       + `hashtable`: transactional hash table implementation
+       + `queue`: transactional work-queue implementation
+       + `list`: simple growable list implementation
+       + `marker`, `prof`: mechanism to record events
+       + `misc`: mostly debug and testing interface
+       + `pagecopy`: fast copy implementation for pages
+       + `prebuilt`: logic for PyPy's prebuilt objects
+       + `setup`: setup code
+       
+       
+
+### Running Tests
+
+Tests are written in Python that calls the C-library through CFFI (Python package).
+
+ 1. install `pytest` and `cffi` packages for Python (via `pip`)
+ 2. running py.test in c8/test should run all the tests (alternatively, the
+    PyPy-checkout has a pytest.py script in its root-folder, which should work
+    too)
+
+### Running Demos
+
+Demos are small C programs that use the STMGC library directly. They sometimes
+expose real data-races that the sequential Python tests cannot expose.
+
+ 1. for example: `make build-demo_random2`
+ 2. then run `./build-demo_random2`
+
+
+## Building PyPy-STM
+
+The STM branch of PyPy contains a *copy* of the STMGC library. After changes to
+STMGC, run the `import_stmgc.py` script in `/rpython/translator/stm/`. In the
+following, `/` is the root of your PyPy checkout.
+
+ 1. The Makefile expects a `gcc-seg-gs` executable to be on the `$PATH`. This
+    should be a GCC that is either patched or a wrapper to GCC 6.1 that passes
+    the necessary options. In my case, this is a script that points to my custom
+    build of GCC with the following content:
+    
+    ```bash
+    #!/bin/bash
+    BUILD=/home/remi/work/bin/gcc-build
+    exec $BUILD/gcc/xgcc -B $BUILD/gcc -fno-ivopts -fno-tree-vectorize -fno-tree-loop-distribute-patterns "$@"
+    ```
+    
+ 2. `cd /pypy/goal/`
+ 
+ 3. A script to translate PyPy-STM (adapt all paths):
+ 
+    ```bash
+    #!/bin/bash
+    export PYPY_USESSION_KEEP=200
+    export PYPY_USESSION_DIR=~/pypy-usession
+
+    STM=--stm #--stm
+    JIT=-Ojit #-Ojit #-O2
+    VERSION=$(hg id -i)
+    time  ionice -c3 pypy ~/pypy_dir/rpython/bin/rpython --no-shared --source $STM $JIT  targetpypystandalone.py
+    # --no-allworkingmodules
+
+    notify-send "PyPy" "C source generated."
+
+    cd ~/pypy-usession/usession-$(hg branch)-remi/testing_1/
+    ionice -c3 make -Bj4
+
+    TIME=$(date +%y-%m-%d-%H:%M)
+    cp pypy-c ~/pypy_dir/pypy/goal/pypy-c-$STM-$JIT-$VERSION-$TIME
+    cp pypy-c ~/pypy_dir/pypy/goal/pypy-c
+
+    notify-send "PyPy" "Make finished."
+    ```
+    
+    The usession-folder will keep the produced C source files. You will need
+    them whenever you do a change to the STMGC library only (no need to
+    retranslate the full PyPy). In that case:
+    
+     1. Go to `~/pypy-usession/usession-stmgc-c8-$USER/testing_1/`
+     2. `make clean && make -j8` will rebuild all C sources
+        
+        Faster alternative that works in most cases: `rm ../module_cache/*.o`
+        instead of `make clean`. This will remove the compiled STMGC files,
+        forcing a rebuild from the *copy* in the `/rpython/translator/stm`
+        folder.
+        
+ 4. The script puts a `pypy-c` into `/pypy/goal/` that should be ready to run.
+
+
+
+
+

From pypy.commits at gmail.com  Thu Feb  9 04:59:25 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 01:59:25 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: carry memory and native
 parameters through _vmprof to rvmprof.py,
 comment to only call vmprof_execute_code once
Message-ID: <589c3d7d.d3811c0a.f875b.8f6c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90016:cd5eb404324c
Date: 2017-02-09 10:58 +0100
http://bitbucket.org/pypy/pypy/changeset/cd5eb404324c/

Log:	carry memory and native parameters through _vmprof to rvmprof.py,
	comment to only call vmprof_execute_code once

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -50,8 +50,8 @@
     return OperationError(w_VMProfError, space.wrap(e.msg))
 
 
- at unwrap_spec(fileno=int, period=float)
-def enable(space, fileno, period):
+ at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int)
+def enable(space, fileno, period, memory, lines, native):
     """Enable vmprof.  Writes go to the given 'fileno', a file descriptor
     opened for writing.  *The file descriptor must remain open at least
     until disable() is called.*
@@ -65,7 +65,7 @@
     #                          "with vmprof will crash"),
     #               space.w_RuntimeWarning)
     try:
-        rvmprof.enable(fileno, period)
+        rvmprof.enable(fileno, period, memory, native)
     except rvmprof.VMProfError as e:
         raise VMProfError(space, e)
 
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,5 +1,6 @@
 import sys, os
-from rpython.rlib.objectmodel import specialize, we_are_translated, not_rpython
+from rpython.rlib.objectmodel import (specialize, we_are_translated,
+        not_rpython, we_are_translated_to_c)
 from rpython.rlib import jit, rposix, rgc
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
@@ -177,6 +178,10 @@
     arguments given to the decorated function.
 
     'result_class' is ignored (backward compatibility).
+
+    NOTE Native profiling: this function can only be called once during
+    translation. To remove this restriction, one needs to extend the macro
+    IS_VMPROF_EVAL in the repo vmprof/vmprof-python.git.
     """
     if _hack_update_stack_untranslated:
         from rpython.rtyper.annlowlevel import llhelper
@@ -209,8 +214,8 @@
             # have added a stack entry, but the function __vmprof_eval_vmprof
             # is not entered. This behaviour will swallow one Python stack frame
             #
-            # Current fix: vmprof will discard this sample. those happen
-            # very infrequent
+            # Current fix: vmprof will discard this sample. (happens
+            # very infrequently)
             #
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -194,7 +194,7 @@
         //}
 
 
-        if ((void*)pip.start_ip == (void*)VMPROF_EVAL()) {
+        if (IS_VMPROF_EVAL((void*)pip.start_ip)) {
             // yes we found one stack entry of the python frames!
 #ifndef RPYTHON_VMPROF
             unw_word_t rbx = 0;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -49,7 +49,7 @@
 // the return type will default to int
 typedef long Signed;
 RPY_EXTERN Signed __vmprof_eval_vmprof();
-#define VMPROF_EVAL() __vmprof_eval_vmprof
+#define IS_VMPROF_EVAL(PTR) PTR == (void*)__vmprof_eval_vmprof
 #else
 #define RPY_EXTERN
 // for cpython
@@ -63,6 +63,7 @@
 #define FRAME_CODE(f) f->f_code
 PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag);
 #define VMPROF_EVAL() vmprof_eval
+#define IS_VMPROF_EVAL(PTR) PTR == (void*)vmprof_eval
 #endif
 
 
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -175,7 +175,12 @@
     if (stack == NULL) {
         stack = get_vmprof_stack();
     }
+    int enabled = vmp_native_enabled();
+    vmp_native_disable();
     n = get_stack_trace(stack, result_p, result_length - 2, pc);
+    if (enabled) {
+        vmp_native_enable();
+    }
     return (intptr_t)n;
 }
 #endif
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -4,6 +4,8 @@
 from rpython.translator.c.test.test_genc import compile
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.nonconst import NonConstant
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rtyper.lltypesystem import rffi, lltype
 
 
 def test_vmprof_execute_code_1():
@@ -148,8 +150,6 @@
         assert os.path.exists(tmpfilename)
         os.unlink(tmpfilename)
 
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import rffi, lltype
 def test_native():
     eci = ExternalCompilationInfo(compile_extra=['-g','-O1'],
             separate_module_sources=["""

From pypy.commits at gmail.com  Thu Feb  9 05:07:04 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 09 Feb 2017 02:07:04 -0800 (PST)
Subject: [pypy-commit] stmgc default: try different code block syntax...
Message-ID: <589c3f48.04ba1c0a.47e8a.9dba@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2007:569798ed0200
Date: 2017-02-09 11:06 +0100
http://bitbucket.org/pypy/stmgc/changeset/569798ed0200/

Log:	try different code block syntax...

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -105,38 +105,37 @@
     the necessary options. In my case, this is a script that points to my custom
     build of GCC with the following content:
     
-    ```bash
-    #!/bin/bash
-    BUILD=/home/remi/work/bin/gcc-build
-    exec $BUILD/gcc/xgcc -B $BUILD/gcc -fno-ivopts -fno-tree-vectorize -fno-tree-loop-distribute-patterns "$@"
-    ```
+        :::bash
+        #!/bin/bash
+        BUILD=/home/remi/work/bin/gcc-build
+        exec $BUILD/gcc/xgcc -B $BUILD/gcc -fno-ivopts -fno-tree-vectorize -fno-tree-loop-distribute-patterns "$@"
+    
     
  2. `cd /pypy/goal/`
  
  3. A script to translate PyPy-STM (adapt all paths):
  
-    ```bash
-    #!/bin/bash
-    export PYPY_USESSION_KEEP=200
-    export PYPY_USESSION_DIR=~/pypy-usession
-
-    STM=--stm #--stm
-    JIT=-Ojit #-Ojit #-O2
-    VERSION=$(hg id -i)
-    time  ionice -c3 pypy ~/pypy_dir/rpython/bin/rpython --no-shared --source $STM $JIT  targetpypystandalone.py
-    # --no-allworkingmodules
-
-    notify-send "PyPy" "C source generated."
-
-    cd ~/pypy-usession/usession-$(hg branch)-remi/testing_1/
-    ionice -c3 make -Bj4
-
-    TIME=$(date +%y-%m-%d-%H:%M)
-    cp pypy-c ~/pypy_dir/pypy/goal/pypy-c-$STM-$JIT-$VERSION-$TIME
-    cp pypy-c ~/pypy_dir/pypy/goal/pypy-c
-
-    notify-send "PyPy" "Make finished."
-    ```
+        :::bash
+        #!/bin/bash
+        export PYPY_USESSION_KEEP=200
+        export PYPY_USESSION_DIR=~/pypy-usession
+        
+        STM=--stm #--stm
+        JIT=-Ojit #-Ojit #-O2
+        VERSION=$(hg id -i)
+        time  ionice -c3 pypy ~/pypy_dir/rpython/bin/rpython --no-shared --source $STM $JIT  targetpypystandalone.py
+        # --no-allworkingmodules
+        
+        notify-send "PyPy" "C source generated."
+        
+        cd ~/pypy-usession/usession-$(hg branch)-remi/testing_1/
+        ionice -c3 make -Bj4
+        
+        TIME=$(date +%y-%m-%d-%H:%M)
+        cp pypy-c ~/pypy_dir/pypy/goal/pypy-c-$STM-$JIT-$VERSION-$TIME
+        cp pypy-c ~/pypy_dir/pypy/goal/pypy-c
+        
+        notify-send "PyPy" "Make finished."
     
     The usession-folder will keep the produced C source files. You will need
     them whenever you do a change to the STMGC library only (no need to

From pypy.commits at gmail.com  Thu Feb  9 05:14:40 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 09 Feb 2017 02:14:40 -0800 (PST)
Subject: [pypy-commit] stmgc default: cosmetics
Message-ID: <589c4110.10941c0a.3f86e.9b39@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2008:c15befd75971
Date: 2017-02-09 11:14 +0100
http://bitbucket.org/pypy/stmgc/changeset/c15befd75971/

Log:	cosmetics

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@
 The library interface is in `c8/stmgc.h`.
 
 Progress (these revisions are roughly stable versions, pick the last one):
+
  - 3af462f
 
 Run tests with `py.test`.
@@ -46,35 +47,41 @@
 
  - `stmgc.h`: the main header file for the library
  - `stm/`: 
-    - For the GC part: 
-       + `nursery`: minor collection
-       + `gcpage`: major collection
-       + `largemalloc`, `smallmalloc`: object allocation
-       + `finalizer`: object finalizer support
-       + `weakref`: weak references support
-    - For the STM part:
-       + `core`: commit, abort, barrier logic of STM
-       + `sync`: segment management and thread support
-       + `pages`: management of page metadata
-       + `signal_handler`: manages pages together with `pages`
-       + `locks`: a set of locks to protect segments
-       + `rewind_setjmp`: setjmp/longjmp implementation that supports arbitrary rollback
-       + `forksupport`: support for forking an STM process
-       + `extra`: on-commit and on-abort callback mechanism
-       + `detach`: transaction detach mechanism (optimised transactional zones)
-    - Misc:
-       + `fprintcolor`: colourful debug output
-       + `hash_id`: PyPy-compatible identity and identity-hash functionality
-       + `hashtable`: transactional hash table implementation
-       + `queue`: transactional work-queue implementation
-       + `list`: simple growable list implementation
-       + `marker`, `prof`: mechanism to record events
-       + `misc`: mostly debug and testing interface
-       + `pagecopy`: fast copy implementation for pages
-       + `prebuilt`: logic for PyPy's prebuilt objects
-       + `setup`: setup code
+ 
+    For the GC part: 
+    
+     - `nursery`: minor collection
+     - `gcpage`: major collection
+     - `largemalloc`, `smallmalloc`: object allocation
+     - `finalizer`: object finalizer support
+     - `weakref`: weak references support
+     
+    For the STM part:
+    
+     - `core`: commit, abort, barrier logic of STM
+     - `sync`: segment management and thread support
+     - `pages`: management of page metadata
+     - `signal_handler`: manages pages together with `pages`
+     - `locks`: a set of locks to protect segments
+     - `rewind_setjmp`: setjmp/longjmp implementation that supports arbitrary rollback
+     - `forksupport`: support for forking an STM process
+     - `extra`: on-commit and on-abort callback mechanism
+     - `detach`: transaction detach mechanism (optimised transactional zones)
+     
+    Misc:
+    
+     - `fprintcolor`: colourful debug output
+     - `hash_id`: PyPy-compatible identity and identity-hash functionality
+     - `hashtable`: transactional hash table implementation
+     - `queue`: transactional work-queue implementation
+     - `list`: simple growable list implementation
+     - `marker`, `prof`: mechanism to record events
+     - `misc`: mostly debug and testing interface
+     - `pagecopy`: fast copy implementation for pages
+     - `prebuilt`: logic for PyPy's prebuilt objects
+     - `setup`: setup code
        
-       
+
 
 ### Running Tests
 

From pypy.commits at gmail.com  Thu Feb  9 05:35:01 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 02:35:01 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: check to not register more than
 one entry in _code_classes
Message-ID: <589c45d5.c69bdf0a.2cedd.0d2a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90017:b3fd68be4074
Date: 2017-02-09 11:12 +0100
http://bitbucket.org/pypy/pypy/changeset/b3fd68be4074/

Log:	check to not register more than one entry in _code_classes

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -94,6 +94,10 @@
         """
         if CodeClass in self._code_classes:
             return
+        assert len(self._code_classes) == 0, \
+                "native profiling currently prohibits " \
+                "to have more than one code class. see comment "\
+                "in vmprof_execute_code for details"
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
@@ -180,8 +184,9 @@
     'result_class' is ignored (backward compatibility).
 
     NOTE Native profiling: this function can only be called once during
-    translation. To remove this restriction, one needs to extend the macro
-    IS_VMPROF_EVAL in the repo vmprof/vmprof-python.git.
+    translation. It would require the macro IS_VMPROF_EVAL to check
+    the IP agains several (not one) symbols. The current symbol is called
+    __vmprof_eval_vmprof.
     """
     if _hack_update_stack_untranslated:
         from rpython.rtyper.annlowlevel import llhelper

From pypy.commits at gmail.com  Thu Feb  9 05:35:04 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 02:35:04 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: revert that check,
 does not work for tests
Message-ID: <589c45d8.c3161c0a.94ff4.a43c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90018:7c464bc97d6f
Date: 2017-02-09 11:34 +0100
http://bitbucket.org/pypy/pypy/changeset/7c464bc97d6f/

Log:	revert that check, does not work for tests

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -79,6 +79,9 @@
 _ssl
     libssl
 
+_vmprof
+    libunwind
+
 Make sure to have these libraries (with development headers) installed
 before building PyPy, otherwise the resulting binary will not contain
 these modules.  Furthermore, the following libraries should be present
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -94,10 +94,11 @@
         """
         if CodeClass in self._code_classes:
             return
-        assert len(self._code_classes) == 0, \
-                "native profiling currently prohibits " \
-                "to have more than one code class. see comment "\
-                "in vmprof_execute_code for details"
+        # nope does not work for our tests
+        # assert len(self._code_classes) == 0, \
+        #         "native profiling currently prohibits " \
+        #         "to have more than one code class. see comment "\
+        #         "in vmprof_execute_code for details"
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -44,12 +44,19 @@
 #define PY_THREAD_STATE_T void
 #define FRAME_STEP(f) f->next
 #define FRAME_CODE(f) f->
-// Is there is a way to tell the compiler
-// that this prototype can have ANY return value. Just removing
-// the return type will default to int
-typedef long Signed;
-RPY_EXTERN Signed __vmprof_eval_vmprof();
-#define IS_VMPROF_EVAL(PTR) PTR == (void*)__vmprof_eval_vmprof
+
+#ifdef RPYTHON_LL2CTYPES
+#  define IS_VMPROF_EVAL(PTR) 0
+#else
+    // Is there is a way to tell the compiler
+    // that this prototype can have ANY return value. Just removing
+    // the return type will default to int
+    typedef long Signed;
+    RPY_EXTERN Signed __vmprof_eval_vmprof();
+#  define IS_VMPROF_EVAL(PTR) PTR == (void*)__vmprof_eval_vmprof
+#endif
+
+
 #else
 #define RPY_EXTERN
 // for cpython

From pypy.commits at gmail.com  Thu Feb  9 05:51:25 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 02:51:25 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: extend the comment on
 vmprof_execute_code, add more parameters to _vmprof.enable in the tests
Message-ID: <589c49ad.63afdf0a.bc7c.0f3b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90019:a68ddcd69e6c
Date: 2017-02-09 11:50 +0100
http://bitbucket.org/pypy/pypy/changeset/a68ddcd69e6c/

Log:	extend the comment on vmprof_execute_code, add more parameters to
	_vmprof.enable in the tests

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -48,7 +48,7 @@
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
-        _vmprof.enable(tmpfileno, 0.01)
+        _vmprof.enable(tmpfileno, 0.01, 0, 0, 0)
         _vmprof.disable()
         s = open(self.tmpfilename, 'rb').read()
         no_of_codes = count(s)
@@ -61,7 +61,7 @@
 
         gc.collect()
         gc.collect()
-        _vmprof.enable(tmpfileno2, 0.01)
+        _vmprof.enable(tmpfileno2, 0.01, 0, 0, 0)
 
         exec """def foo2():
             pass
@@ -76,9 +76,9 @@
 
     def test_enable_ovf(self):
         import _vmprof
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 0)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 0, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300, 0, 0, 0)
         NaN = (1e300*1e300) / (1e300*1e300)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN, 0, 0, 0)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -94,11 +94,6 @@
         """
         if CodeClass in self._code_classes:
             return
-        # nope does not work for our tests
-        # assert len(self._code_classes) == 0, \
-        #         "native profiling currently prohibits " \
-        #         "to have more than one code class. see comment "\
-        #         "in vmprof_execute_code for details"
         CodeClass._vmprof_unique_id = 0     # default value: "unknown"
         immut = CodeClass.__dict__.get('_immutable_fields_', [])
         CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id']
@@ -184,10 +179,21 @@
 
     'result_class' is ignored (backward compatibility).
 
-    NOTE Native profiling: this function can only be called once during
-    translation. It would require the macro IS_VMPROF_EVAL to check
-    the IP agains several (not one) symbols. The current symbol is called
-    __vmprof_eval_vmprof.
+    ====================================
+    TRANSLATION NOTE CALL THIS ONLY ONCE
+    ====================================
+
+    This function can only be called once during translation.
+    It generates a C function called __vmprof_eval_vmprof which is used by
+    the C source code as an extern function. This is necessary while walking
+    the native stack. If you see __vmprof_eval_vmprof defined twice during
+    translation, read on:
+
+    To remove this restriction do the following:
+
+    *) Extend the macro IS_VMPROF_EVAL in the vmprof source repo to check several
+       sybmols.
+    *) Give each function provided to this decorator a unique name in C
     """
     if _hack_update_stack_untranslated:
         from rpython.rtyper.annlowlevel import llhelper

From pypy.commits at gmail.com  Thu Feb  9 06:23:11 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 03:23:11 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: apply changes made in pypy source
 repo, and never ported back to vmprof source
Message-ID: <589c511f.018e1c0a.234ed.b73c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90020:dffb13845738
Date: 2017-02-09 12:22 +0100
http://bitbucket.org/pypy/pypy/changeset/dffb13845738/

Log:	apply changes made in pypy source repo, and never ported back to
	vmprof source

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -24,10 +24,11 @@
             i += 5 * WORD # header
             assert s[i    ] == '\x05'    # MARKER_HEADER
             assert s[i + 1] == '\x00'    # 0
-            assert s[i + 2] == '\x02'    # VERSION_THREAD_ID
-            assert s[i + 3] == chr(4)    # len('pypy')
-            assert s[i + 4: i + 8] == 'pypy'
-            i += 8
+            assert s[i + 2] == '\x06'    # VERSION_TIMESTAMP
+            assert s[i + 3] == '\x08'    # PROFILE_RPYTHON
+            assert s[i + 4] == chr(4)    # len('pypy')
+            assert s[i + 5: i + 9] == 'pypy'
+            i += 9
             while i < len(s):
                 if s[i] == '\x03':
                     break
@@ -41,6 +42,8 @@
                     _, size = struct.unpack("ll", s[i:i + 2 * WORD])
                     count += 1
                     i += 2 * WORD + size
+                elif s[i] == '\x06':
+                    i += 8+8+8
                 else:
                     raise AssertionError(ord(s[i]))
             return count
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -43,7 +43,7 @@
 }
 
 
-""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read(), include_dirs=[str(srcdir)])
+""" + open(str(srcdir.join("shared/vmprof_get_custom_offset.h"))).read(), include_dirs=[str(srcdir)])
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -185,15 +185,16 @@
 
     This function can only be called once during translation.
     It generates a C function called __vmprof_eval_vmprof which is used by
-    the C source code as an extern function. This is necessary while walking
-    the native stack. If you see __vmprof_eval_vmprof defined twice during
+    the vmprof C source code and is bound as an extern function.
+    This is necessary while walking the native stack.
+    If you see __vmprof_eval_vmprof defined twice during
     translation, read on:
 
     To remove this restriction do the following:
 
     *) Extend the macro IS_VMPROF_EVAL in the vmprof source repo to check several
        sybmols.
-    *) Give each function provided to this decorator a unique name in C
+    *) Give each function provided to this decorator a unique symbol name in C
     """
     if _hack_update_stack_untranslated:
         from rpython.rtyper.annlowlevel import llhelper
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -60,8 +60,9 @@
 char *vmprof_init(int fd, double interval, int memory,
                   int lines, const char *interp_name, int native)
 {
-    if (interval < 1e-6 || interval >= 1.0)
+    if (!(interval >= 1e-6 && interval < 1.0)) {   /* also if it is NaN */
         return "bad value for 'interval'";
+    }
     prepare_interval_usec = (int)(interval * 1000000.0);
 
     if (prepare_concurrent_bufs() < 0)
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
@@ -1,120 +1,53 @@
 #pragma once
 
-#ifdef PYPY_JIT_CODEMAP
 void *pypy_find_codemap_at_addr(long addr, long *start_addr);
 long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
                                 long *current_pos_addr);
-long pypy_jit_stack_depth_at_loc(long loc);
-#endif
 
+#define MAX_INLINE_DEPTH  384
 
-#ifdef CPYTHON_GET_CUSTOM_OFFSET
-static void *tramp_start, *tramp_end;
-#endif
-
-
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-
-#if defined(PYPY_JIT_CODEMAP)
-
-    intptr_t ip_l = (intptr_t)ip;
-    return pypy_jit_stack_depth_at_loc(ip_l);
-
-#elif defined(CPYTHON_GET_CUSTOM_OFFSET)
-
-    if (ip >= tramp_start && ip <= tramp_end) {
-        // XXX the return value is wrong for all the places before push and
-        //     after pop, fix
-        void *bp;
-        void *sp;
-
-        /* This is a stage2 trampoline created by hotpatch:
-
-               push   %rbx
-               push   %rbp
-               mov    %rsp,%rbp
-               and    $0xfffffffffffffff0,%rsp   // make sure the stack is aligned
-               movabs $0x7ffff687bb10,%rbx
-               callq  *%rbx
-               leaveq 
-               pop    %rbx
-               retq   
-
-           the stack layout is like this:
-
-               +-----------+                      high addresses
-               | ret addr  |
-               +-----------+
-               | saved rbx |   start of the function frame
-               +-----------+
-               | saved rbp |
-               +-----------+
-               | ........  |   <-- rbp
-               +-----------+                      low addresses
-
-           So, the trampoline frame starts at rbp+16, and the return address,
-           is at rbp+24.  The vmprof API requires us to return the offset of
-           the frame relative to sp, hence we have this weird computation.
-
-           XXX (antocuni): I think we could change the API to return directly
-           the frame address instead of the offset; however, this require a
-           change in the PyPy code too
-        */
-
-        unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
-        unw_get_reg (cp, UNW_X86_64_RBP, (unw_word_t*)&bp);
-        return bp+16+8-sp;
-    }
-    return -1;
-
-#else
-
-    return -1;
-
-#endif
-}
-
-static long vmprof_write_header_for_jit_addr(void **result, long n,
-                                             void *ip, int max_depth)
+static long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
+                                             intptr_t addr, int max_depth)
 {
 #ifdef PYPY_JIT_CODEMAP
     void *codemap;
     long current_pos = 0;
-    intptr_t id;
+    intptr_t ident, local_stack[MAX_INLINE_DEPTH];
+    long m;
     long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    void *tmp;
 
     codemap = pypy_find_codemap_at_addr(addr, &start_addr);
-    if (codemap == NULL)
-        // not a jit code at all
+    if (codemap == NULL || n >= max_depth - 2)
+        // not a jit code at all or almost max depth
         return n;
 
     // modify the last entry to point to start address and not the random one
     // in the middle
-    result[n - 1] = (void*)start_addr;
-    result[n] = (void*)2;
-    n++;
-    start = n;
-    while (n < max_depth) {
-        id = pypy_yield_codemap_at_addr(codemap, addr, &current_pos);
-        if (id == -1)
+    result[n++] = VMPROF_ASSEMBLER_TAG;
+    result[n++] = start_addr;
+
+    // build the list of code idents corresponding to the current
+    // position inside this particular piece of assembler.  If (very
+    // unlikely) we get more than MAX_INLINE_DEPTH recursion levels
+    // all inlined inside this single piece of assembler, then stop:
+    // there will be some missing frames then.  Otherwise, we need to
+    // first collect 'local_stack' and then write it to 'result' in the
+    // opposite order, stopping at 'max_depth'.  Previous versions of
+    // the code would write the oldest calls and then stop---whereas
+    // what we really need it to write the newest calls and then stop.
+    m = 0;
+    while (m < MAX_INLINE_DEPTH) {
+        ident = pypy_yield_codemap_at_addr(codemap, addr, &current_pos);
+        if (ident == -1)
             // finish
             break;
-        if (id == 0)
+        if (ident == 0)
             continue; // not main codemap
-        result[n++] = (void *)id;
+        local_stack[m++] = ident;
     }
-    k = 0;
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k - 1];
-        result[n - k - 1] = tmp;
-        k++;
-    }
-    if (n < max_depth) {
-        result[n++] = (void*)3;
+    while (m > 0 && n < max_depth) {
+        result[n++] = VMPROF_JITTED_TAG;
+        result[n++] = local_stack[--m];
     }
 #endif
     return n;

From pypy.commits at gmail.com  Thu Feb  9 06:29:11 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 03:29:11 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: remove old file
Message-ID: <589c5287.8c7e1c0a.23119.b8ea@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90021:42fe1afeea56
Date: 2017-02-09 12:23 +0100
http://bitbucket.org/pypy/pypy/changeset/42fe1afeea56/

Log:	remove old file

diff --git a/rpython/rlib/rvmprof/src/vmprof_unwind.h b/rpython/rlib/rvmprof/src/vmprof_unwind.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/vmprof_unwind.h
+++ /dev/null
@@ -1,89 +0,0 @@
-#include <stddef.h>
-#include <stdint.h>
-#include <ucontext.h>
-
-// copied from libunwind.h
-
-typedef enum
-  {
-    UNW_X86_64_RAX,
-    UNW_X86_64_RDX,
-    UNW_X86_64_RCX,
-    UNW_X86_64_RBX,
-    UNW_X86_64_RSI,
-    UNW_X86_64_RDI,
-    UNW_X86_64_RBP,
-    UNW_X86_64_RSP,
-    UNW_X86_64_R8,
-    UNW_X86_64_R9,
-    UNW_X86_64_R10,
-    UNW_X86_64_R11,
-    UNW_X86_64_R12,
-    UNW_X86_64_R13,
-    UNW_X86_64_R14,
-    UNW_X86_64_R15,
-    UNW_X86_64_RIP,
-#ifdef CONFIG_MSABI_SUPPORT
-    UNW_X86_64_XMM0,
-    UNW_X86_64_XMM1,
-    UNW_X86_64_XMM2,
-    UNW_X86_64_XMM3,
-    UNW_X86_64_XMM4,
-    UNW_X86_64_XMM5,
-    UNW_X86_64_XMM6,
-    UNW_X86_64_XMM7,
-    UNW_X86_64_XMM8,
-    UNW_X86_64_XMM9,
-    UNW_X86_64_XMM10,
-    UNW_X86_64_XMM11,
-    UNW_X86_64_XMM12,
-    UNW_X86_64_XMM13,
-    UNW_X86_64_XMM14,
-    UNW_X86_64_XMM15,
-    UNW_TDEP_LAST_REG = UNW_X86_64_XMM15,
-#else
-    UNW_TDEP_LAST_REG = UNW_X86_64_RIP,
-#endif
-
-    /* XXX Add other regs here */
-
-    /* frame info (read-only) */
-    UNW_X86_64_CFA,
-
-    UNW_TDEP_IP = UNW_X86_64_RIP,
-    UNW_TDEP_SP = UNW_X86_64_RSP,
-    UNW_TDEP_BP = UNW_X86_64_RBP,
-    UNW_TDEP_EH = UNW_X86_64_RAX
-  }
-x86_64_regnum_t;
-
-typedef uint64_t unw_word_t;
-
-#define UNW_TDEP_CURSOR_LEN 127
-
-typedef struct unw_cursor
-  {
-    unw_word_t opaque[UNW_TDEP_CURSOR_LEN];
-  } unw_cursor_t;
-
-#define UNW_REG_IP UNW_X86_64_RIP
-#define UNW_REG_SP UNW_X86_64_RSP
-
-typedef ucontext_t unw_context_t;
-
-typedef struct unw_proc_info
-  {
-    unw_word_t start_ip;	/* first IP covered by this procedure */
-    unw_word_t end_ip;		/* first IP NOT covered by this procedure */
-    unw_word_t lsda;		/* address of lang.-spec. data area (if any) */
-    unw_word_t handler;		/* optional personality routine */
-    unw_word_t gp;		/* global-pointer value for this procedure */
-    unw_word_t flags;		/* misc. flags */
-
-    int format;			/* unwind-info format (arch-specific) */
-    int unwind_info_size;	/* size of the information (if applicable) */
-    void *unwind_info;		/* unwind-info (arch-specific) */
-  } unw_proc_info_t;
-
-// end of copy
-

From pypy.commits at gmail.com  Thu Feb  9 06:29:13 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 03:29:13 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: handle all vmprof TAGS in the
 test that decodes the logfile
Message-ID: <589c5289.0fb81c0a.47e09.b662@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90022:f6addbdf4889
Date: 2017-02-09 12:28 +0100
http://bitbucket.org/pypy/pypy/changeset/f6addbdf4889/

Log:	handle all vmprof TAGS in the test that decodes the logfile

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -43,7 +43,16 @@
                     count += 1
                     i += 2 * WORD + size
                 elif s[i] == '\x06':
-                    i += 8+8+8
+                    print(s[i:i+24])
+                    i += 1+8+8+8
+                elif s[i] == '\x07':
+                    i += 1
+                    # skip string
+                    size, = struct.unpack("l", s[i:i + WORD])
+                    i += WORD+size
+                    # skip string
+                    size, = struct.unpack("l", s[i:i + WORD])
+                    i += WORD+size
                 else:
                     raise AssertionError(ord(s[i]))
             return count

From pypy.commits at gmail.com  Thu Feb  9 06:30:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Feb 2017 03:30:03 -0800 (PST)
Subject: [pypy-commit] cffi default: The API mode is faster. Put this
 forward more.
Message-ID: <589c52bb.125e1c0a.46bd3.b96e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2888:58b6002275b8
Date: 2017-02-09 12:29 +0100
http://bitbucket.org/cffi/cffi/changeset/58b6002275b8/

Log:	The API mode is faster. Put this forward more.

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -48,9 +48,9 @@
 
 .. _out-of-line-api:
 
-* The **"out-of-line", "API mode"** gives you the most flexibility to
-  access a C library at the level of C, instead of at the binary
-  level:
+* The **"out-of-line", "API mode"** gives you the most flexibility
+  and speed to access a C library at the level of C, instead of at the
+  binary level:
 
   .. code-block:: python
 
@@ -278,8 +278,10 @@
 
 ``ffi.dlopen(libpath, [flags])``: this function opens a shared library and
 returns a module-like library object.  Use this when you are fine with
-the limitations of ABI-level access to the system.  In case of doubt, read
-again `ABI versus API`_ in the overview.
+the limitations of ABI-level access to the system (dependency on ABI
+details, getting crashes instead of C compiler errors/warnings, and
+higher overhead to call the C functions).  In case of doubt, read again
+`ABI versus API`_ in the overview.
 
 .. _`ABI versus API`: overview.html#abi-versus-api
 
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -8,7 +8,7 @@
 each with "in-line" or "out-of-line" preparation (or compilation).
 
 The **ABI mode** accesses libraries at the binary level, whereas the
-**API mode** accesses them with a C compiler.  This is described in
+faster **API mode** accesses them with a C compiler.  This is described in
 detail below__.
 
 .. __: `abi-versus-api`_
@@ -51,7 +51,7 @@
 
 If using a C compiler to install your module is an option, it is highly
 recommended to use the API mode described in the next paragraph.  (It is
-also a bit faster at runtime.)
+also faster.)
 
 
 .. _out-of-line-api-level:
@@ -116,7 +116,8 @@
 portable than trying to get the details of the fields of ``struct
 passwd`` exactly right.  Similarly, we declared ``getpwuid()`` as
 taking an ``int`` argument.  On some platforms this might be slightly
-incorrect---but it does not matter.
+incorrect---but it does not matter.  It is also faster than the ABI
+mode.
 
 To integrate it inside a ``setup.py`` distribution with Setuptools:
 
@@ -391,17 +392,25 @@
 --------------
 
 Accessing the C library at the binary level ("ABI") is fraught
-with problems, particularly on non-Windows platforms.  You are not
-meant to access fields by guessing where they are in the structures.
-*The C libraries are typically meant to be used with a C compiler.*
+with problems, particularly on non-Windows platforms.
 
-The "real example" above shows how to do that: this example uses
-``set_source(..., "C source...")`` and never ``dlopen()``.
-When using this approach,
+The most immediate drawback of the ABI level is that calling functions
+needs to go through the very general *libffi* library, which is slow
+(and not always perfectly tested on non-standard platforms).  The API
+mode instead compiles a CPython C wrapper that directly invokes the
+target function.  It is, comparatively, massively faster (and works
+better than libffi ever can).
+
+The more fundamental reason to prefer the API mode is that *the C
+libraries are typically meant to be used with a C compiler.* You are not
+supposed to do things like guess where fields are in the structures.
+The "real example" above shows how CFFI uses a C compiler under the
+hood: this example uses ``set_source(..., "C source...")`` and never
+``dlopen()``.  When using this approach,
 we have the advantage that we can use literally "``...``" at various places in
 the ``cdef()``, and the missing information will be completed with the
-help of the C compiler.  Actually, a single C source file is produced,
-which contains first the "C source" part unmodified, followed by some
+help of the C compiler.  CFFI will turn this into a single C source file,
+which contains the "C source" part unmodified, followed by some
 "magic" C code and declarations derived from the ``cdef()``.  When
 this C file is compiled, the resulting C extension module will contain
 all the information we need---or the C compiler will give warnings or

From pypy.commits at gmail.com  Thu Feb  9 07:33:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Feb 2017 04:33:40 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <589c61a4.10941c0a.3f86e.d440@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r852:a259f7482c20
Date: 2017-02-09 13:33 +0100
http://bitbucket.org/pypy/pypy.org/changeset/a259f7482c20/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66570 of $105000 (63.4%)
+   $66589 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2399 left
+    we have $2417 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Thu Feb  9 09:06:31 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 06:06:31 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: merge default
Message-ID: <589c7767.84301c0a.10d3d.af8a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90024:1ae8adccedfb
Date: 2017-02-09 14:53 +0100
http://bitbucket.org/pypy/pypy/changeset/1ae8adccedfb/

Log:	merge default

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -462,7 +462,12 @@
         field or array item in the structure or array, recursively in
         case of nested structures.
         """
-        ctype = self._backend.typeof(cdata)
+        try:
+            ctype = self._backend.typeof(cdata)
+        except TypeError:
+            if '__addressof__' in type(cdata).__dict__:
+                return type(cdata).__addressof__(cdata, *fields_or_indexes)
+            raise
         if fields_or_indexes:
             ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
         else:
@@ -775,10 +780,7 @@
         key = 'function ' + name
         tp, _ = ffi._parser._declarations[key]
         BType = ffi._get_cached_btype(tp)
-        try:
-            value = backendlib.load_function(BType, name)
-        except KeyError as e:
-            raise AttributeError('%s: %s' % (name, e))
+        value = backendlib.load_function(BType, name)
         library.__dict__[name] = value
     #
     def accessor_variable(name):
@@ -791,6 +793,21 @@
             lambda self: read_variable(BType, name),
             lambda self, value: write_variable(BType, name, value)))
     #
+    def addressof_var(name):
+        try:
+            return addr_variables[name]
+        except KeyError:
+            with ffi._lock:
+                if name not in addr_variables:
+                    key = 'variable ' + name
+                    tp, _ = ffi._parser._declarations[key]
+                    BType = ffi._get_cached_btype(tp)
+                    if BType.kind != 'array':
+                        BType = model.pointer_cache(ffi, BType)
+                    p = backendlib.load_function(BType, name)
+                    addr_variables[name] = p
+            return addr_variables[name]
+    #
     def accessor_constant(name):
         raise NotImplementedError("non-integer constant '%s' cannot be "
                                   "accessed from a dlopen() library" % (name,))
@@ -800,6 +817,7 @@
     #
     accessors = {}
     accessors_version = [False]
+    addr_variables = {}
     #
     def update_accessors():
         if accessors_version[0] is ffi._cdef_version:
@@ -850,6 +868,18 @@
             with ffi._lock:
                 update_accessors()
                 return accessors.keys()
+        def __addressof__(self, name):
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            make_accessor(name)
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            raise AttributeError("cffi library has no function or "
+                                 "global variable named '%s'" % (name,))
     #
     if libname is not None:
         try:
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.11
+Version: 0.4.12
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.11"
+__version__ = "0.4.12"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -14,7 +14,7 @@
 
 * Write them in C++ and bind them through  :doc:`cppyy <cppyy>` using Cling.
 
-* Write them in as `RPython mixed modules`_.
+* Write them as `RPython mixed modules`_.
 
 
 CFFI
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -38,6 +38,21 @@
 .. _mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 
 
+Explicit typing in RPython
+--------------------------
+
+RPython is mostly based around type inference, but there are many cases where
+specifying types explicitly is useful. We would like to be able to optionally 
+specify the exact types of the arguments to any function. We already have
+solutions in that space, ``@rpython.rlib.objectmodel.enforceargs`` and
+``@rpython.rlib.signature.signature``, but they are inconvenient and limited. 
+For instance, they do not easily allow to express the type "dict with ints as
+keys and lists of instances of Foo as values".
+
+Additionally, we would like to be able to specify the types of instance
+attributes. Unlike the function case, this is likely to require some
+refactoring of the annotator. 
+
 Make bytearray type fast
 ------------------------
 
@@ -176,28 +191,33 @@
 * `hg`
 
 
-Optimising cpyext (CPython C-API compatibility layer)
------------------------------------------------------
-
-A lot of work has gone into PyPy's implementation of CPython's C-API over
-the last years to let it reach a practical level of compatibility, so that
-C extensions for CPython work on PyPy without major rewrites. However,
-there are still many edges and corner cases where it misbehaves.
-
-The objective of this project is to fix bugs in cpyext and to optimise
-several performance critical parts of it, such as the reference counting
-support and other heavily used C-API functions. The net result would be to
-have CPython extensions run much faster on PyPy than they currently do, or
-to make them work at all if they currently don't. A part of this work would
-be to get cpyext into a shape where it supports running Cython generated
-extensions.
-
 ======================================
 Make more python modules pypy-friendly
 ======================================
 
-Work has been started on a few popular python packages. Here is a partial
-list of good work that needs to be finished:
+A lot of work has gone into PyPy's implementation of CPython's C-API, cpyext,
+over the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However, there
+are still many edges and corner cases where it misbehaves.
+
+For any popular extension that does not already advertise full PyPy
+compatibility, it would thus be useful to take a close look at it in order to
+make it fully compatible with PyPy. The general process is something like:
+
+* Run the extension's tests on PyPy and look at the test failures.
+* Some of the failures may be solved by identifying cases where the extension
+  relies on undocumented or internal details of CPython, and rewriting the
+  relevant code to follow documented best practices. Open issues and send pull
+  requests as appropriate given the extension's development process.
+* Other failures may highlight incompatibilities between cpyext and CPython.
+  Please report them to us and try to fix them.
+* Run benchmarks, either provided by the extension developers or created by
+  you. Any case where PyPy is significantly slower than CPython is to be
+  considered a bug and solved as above.
+
+Alternatively, an approach we used to recommend was to rewrite C extensions
+using more pypy-friendly technologies, e.g. cffi. Here is a partial list of
+good work that needs to be finished:
 
 **matplotlib** https://github.com/mattip/matplotlib
 
@@ -223,5 +243,3 @@
 **pyopengl** https://bitbucket.org/duangle/pyopengl-cffi
 
     Status: unknown
-
-
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -40,25 +40,22 @@
 
     @unwrap_spec(w_ctype=W_CType, name=str)
     def load_function(self, w_ctype, name):
-        from pypy.module._cffi_backend import ctypefunc, ctypeptr, ctypevoid
+        from pypy.module._cffi_backend import ctypeptr, ctypearray
         space = self.space
         #
-        ok = False
-        if isinstance(w_ctype, ctypefunc.W_CTypeFunc):
-            ok = True
-        if (isinstance(w_ctype, ctypeptr.W_CTypePointer) and
-            isinstance(w_ctype.ctitem, ctypevoid.W_CTypeVoid)):
-            ok = True
-        if not ok:
+        if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray):
             raise oefmt(space.w_TypeError,
-                        "function cdata expected, got '%s'", w_ctype.name)
+                        "function or pointer or array cdata expected, got '%s'",
+                        w_ctype.name)
         #
         try:
             cdata = dlsym(self.handle, name)
         except KeyError:
-            raise oefmt(space.w_KeyError,
-                        "function '%s' not found in library '%s'",
+            raise oefmt(space.w_AttributeError,
+                        "function/symbol '%s' not found in library '%s'",
                         name, self.name)
+        if isinstance(w_ctype, ctypearray.W_CTypeArray) and w_ctype.length < 0:
+            w_ctype = w_ctype.ctptr
         return W_CData(space, rffi.cast(rffi.CCHARP, cdata), w_ctype)
 
     @unwrap_spec(w_ctype=W_CType, name=str)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -365,7 +365,7 @@
     x = find_and_load_library(None)
     BVoidP = new_pointer_type(new_void_type())
     assert x.load_function(BVoidP, 'strcpy')
-    py.test.raises(KeyError, x.load_function,
+    py.test.raises(AttributeError, x.load_function,
                    BVoidP, 'xxx_this_function_does_not_exist')
     # the next one is from 'libm', not 'libc', but we assume
     # that it is already loaded too, so it should work
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1227,6 +1227,26 @@
         assert list(a)[:1000] + [0] * (len(a)-1000) == list(b)
         f.close()
 
+    def test_ffi_buffer_comparisons(self):
+        ffi = FFI(backend=self.Backend())
+        ba = bytearray(range(100, 110))
+        assert ba == memoryview(ba)    # justification for the following
+        a = ffi.new("uint8_t[]", list(ba))
+        c = ffi.new("uint8_t[]", [99] + list(ba))
+        try:
+            b_full = ffi.buffer(a)
+            b_short = ffi.buffer(a, 3)
+            b_mid = ffi.buffer(a, 6)
+            b_other = ffi.buffer(c, 6)
+        except NotImplementedError as e:
+            py.test.skip(str(e))
+        else:
+            content = b_full[:]
+            assert content == b_full == ba
+            assert b_other < b_short < b_mid < b_full
+            assert ba > b_mid > ba[0:2]
+            assert b_short != ba[1:4]
+
     def test_array_in_struct(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct foo_s { int len; short data[5]; };")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -193,9 +193,12 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
+        buff1 = ffi.buffer(s)
         t = lib.try_with_value(fnames.index(name), value)
         raw2 = ffi.buffer(t, len(raw1))[:]
         assert raw1 == raw2
+        buff2 = ffi.buffer(t, len(buff1))
+        assert buff1 == buff2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -283,3 +283,21 @@
             assert ret.right == ownlib.right
             assert ret.top == ownlib.top
             assert ret.bottom == ownlib.bottom
+
+    def test_addressof_lib(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("long left; int test_getting_errno(void);")
+        lib = ffi.dlopen(self.module)
+        lib.left = 123456
+        p = ffi.addressof(lib, "left")
+        assert ffi.typeof(p) == ffi.typeof("long *")
+        assert p[0] == 123456
+        p[0] += 1
+        assert lib.left == 123457
+        pfn = ffi.addressof(lib, "test_getting_errno")
+        assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
+        assert pfn == lib.test_getting_errno
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -348,21 +348,15 @@
         w_value = self.getitem(w_key)
         return w_value if w_value is not None else w_default
 
-    @unwrap_spec(defaults_w='args_w')
-    def descr_pop(self, space, w_key, defaults_w):
+    def descr_pop(self, space, w_key, w_default=None):
         """D.pop(k[,d]) -> v, remove specified key and return the
         corresponding value\nIf key is not found, d is returned if given,
         otherwise KeyError is raised
         """
-        len_defaults = len(defaults_w)
-        if len_defaults > 1:
-            raise oefmt(space.w_TypeError,
-                        "pop expected at most 2 arguments, got %d",
-                        1 + len_defaults)
         w_item = self.getitem(w_key)
         if w_item is None:
-            if len_defaults > 0:
-                return defaults_w[0]
+            if w_default is not None:
+                return w_default
             else:
                 space.raise_key_error(w_key)
         else:
@@ -1053,7 +1047,10 @@
         if self.is_correct_type(w_key):
             d = self.unerase(w_dict.dstorage)
             key = self.unwrap(w_key)
-            objectmodel.move_to_end(d, key, last_flag)
+            try:
+                objectmodel.move_to_end(d, key, last_flag)
+            except KeyError:
+                w_dict.space.raise_key_error(w_key)
         else:
             self.switch_to_object_strategy(w_dict)
             w_dict.nondescr_move_to_end(w_dict.space, w_key, last_flag)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -307,6 +307,7 @@
                     assert list(d) == other_keys + [key]
                 else:
                     assert list(d) == [key] + other_keys
+                raises(KeyError, __pypy__.move_to_end, d, key * 3, last=last)
 
     def test_keys(self):
         d = {1: 2, 3: 4}
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -179,7 +179,7 @@
     shutil.copytree(str(basedir.join('lib_pypy')),
                     str(pypydir.join('lib_pypy')),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
-                                           '*.c', '*.o'))
+                                           '*_cffi.c', '*.o'))
     for file in ['README.rst',]:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     for file in ['_testcapimodule.c', '_ctypes_test.c']:
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -783,6 +783,9 @@
     def setdefault(self, key, default):
         return self._dict.setdefault(_r_dictkey(self, key), default)
 
+    def pop(self, key, *default):
+        return self._dict.pop(_r_dictkey(self, key), *default)
+
     def popitem(self):
         dk, value = self._dict.popitem()
         return dk.key, value
diff --git a/rpython/translator/c/src/stacklet/stacklet.c b/rpython/translator/c/src/stacklet/stacklet.c
--- a/rpython/translator/c/src/stacklet/stacklet.c
+++ b/rpython/translator/c/src/stacklet/stacklet.c
@@ -5,8 +5,8 @@
 #include "src/stacklet/stacklet.h"
 
 #include <stddef.h>
-#include <assert.h>
 #include <string.h>
+#include <stdio.h>
 
 /************************************************************
  * platform specific code
@@ -34,7 +34,7 @@
 /************************************************************/
 
 struct stacklet_s {
-    /* The portion of the real stack claimed by this paused tealet. */
+    /* The portion of the real stack claimed by this paused stacklet. */
     char *stack_start;                /* the "near" end of the stack */
     char *stack_stop;                 /* the "far" end of the stack */
 
@@ -69,6 +69,19 @@
     struct stacklet_s *g_target;
 };
 
+#define _check(x)  do { if (!(x)) _check_failed(#x); } while (0)
+
+static void _check_failed(const char *check)
+{
+    fprintf(stderr, "FATAL: stacklet: %s failed\n", check);
+    abort();
+}
+
+static void check_valid(struct stacklet_s *g)
+{
+    _check(g->stack_saved >= 0);
+}
+
 /***************************************************************/
 
 static void g_save(struct stacklet_s* g, char* stop
@@ -96,7 +109,8 @@
      */
     ptrdiff_t sz1 = g->stack_saved;
     ptrdiff_t sz2 = stop - g->stack_start;
-    assert(stop <= g->stack_stop);
+    check_valid(g);
+    _check(stop <= g->stack_stop);
 
     if (sz2 > sz1) {
         char *c = (char *)(g + 1);
@@ -146,11 +160,13 @@
 {
     struct stacklet_s *current = thrd->g_stack_chain_head;
     char *target_stop = g_target->stack_stop;
+    check_valid(g_target);
 
-    /* save and unlink tealets that are completely within
+    /* save and unlink stacklets that are completely within
        the area to clear. */
     while (current != NULL && current->stack_stop <= target_stop) {
         struct stacklet_s *prev = current->stack_prev;
+        check_valid(current);
         current->stack_prev = NULL;
         if (current != g_target) {
             /* don't bother saving away g_target, because
@@ -222,14 +238,16 @@
     struct stacklet_thread_s *thrd = (struct stacklet_thread_s *)rawthrd;
     struct stacklet_s *g = thrd->g_target;
     ptrdiff_t stack_saved = g->stack_saved;
+    check_valid(g);
 
-    assert(new_stack_pointer == g->stack_start);
+    _check(new_stack_pointer == g->stack_start);
 #if STACK_DIRECTION == 0
     memcpy(g->stack_start, g+1, stack_saved);
 #else
     memcpy(g->stack_start - stack_saved, g+1, stack_saved);
 #endif
     thrd->g_current_stack_stop = g->stack_stop;
+    g->stack_saved = -13;   /* debugging */
     free(g);
     return EMPTY_STACKLET_HANDLE;
 }
@@ -250,10 +268,11 @@
         result = run(thrd->g_source, run_arg);
 
         /* Then switch to 'result'. */
+        check_valid(result);
         thrd->g_target = result;
         _stacklet_switchstack(g_destroy_state, g_restore_state, thrd);
 
-        assert(!"stacklet: we should not return here");
+        _check_failed("we should not return here");
         abort();
     }
     /* The second time it returns. */
@@ -287,7 +306,7 @@
                              stacklet_run_fn run, void *run_arg)
 {
     long stackmarker;
-    assert((char *)NULL < (char *)&stackmarker);
+    _check((char *)NULL < (char *)&stackmarker);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
@@ -300,6 +319,7 @@
 {
     long stackmarker;
     stacklet_thread_handle thrd = target->stack_thrd;
+    check_valid(target);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
@@ -310,6 +330,7 @@
 
 void stacklet_destroy(stacklet_handle target)
 {
+    check_valid(target);
     if (target->stack_prev != NULL) {
         /* 'target' appears to be in the chained list 'unsaved_stack',
            so remove it from there.  Note that if 'thrd' was already
@@ -319,12 +340,15 @@
            we don't even read 'stack_thrd', already deallocated. */
         stacklet_thread_handle thrd = target->stack_thrd;
         struct stacklet_s **pp = &thrd->g_stack_chain_head;
-        for (; *pp != NULL; pp = &(*pp)->stack_prev)
+        for (; *pp != NULL; pp = &(*pp)->stack_prev) {
+            check_valid(*pp);
             if (*pp == target) {
                 *pp = target->stack_prev;
                 break;
             }
+        }
     }
+    target->stack_saved = -11;   /* debugging */
     free(target);
 }
 
@@ -334,6 +358,7 @@
   long delta;
   if (context == NULL)
     return ptr;
+  check_valid(context);
   delta = p - context->stack_start;
   if (((unsigned long)delta) < ((unsigned long)context->stack_saved)) {
       /* a pointer to a saved away word */
@@ -345,8 +370,8 @@
       /* out-of-stack pointer!  it's only ok if we are the main stacklet
          and we are reading past the end, because the main stacklet's
          stack stop is not exactly known. */
-      assert(delta >= 0);
-      assert(((long)context->stack_stop) & 1);
+      _check(delta >= 0);
+      _check(((long)context->stack_stop) & 1);
   }
   return ptr;
 }

From pypy.commits at gmail.com  Thu Feb  9 09:06:29 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 06:06:29 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: fix translation issues for
 rvmprof test
Message-ID: <589c7765.348fdf0a.eedc2.5ba8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90023:20988301e8f2
Date: 2017-02-09 14:53 +0100
http://bitbucket.org/pypy/pypy/changeset/20988301e8f2/

Log:	fix translation issues for rvmprof test

diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -10,9 +10,30 @@
 #  include "rvmprof.h"
 #endif
 
+#include "shared/vmprof_get_custom_offset.h"
 #ifdef VMPROF_UNIX
 #include "shared/vmprof_main.h"
 #else
 #include "shared/vmprof_main_win32.h"
 #endif
 
+
+void * g_symbol = 0;
+
+int IS_VMPROF_EVAL(void * ptr)
+{
+#ifdef RPYTHON_LL2CTYPES
+    return 0;
+#else
+
+    if (g_symbol == NULL) {
+        g_symbol = dlsym(RTLD_GLOBAL, "__vmprof_eval_vmprof");
+        if (g_symbol == NULL) {
+            fprintf(stderr, "symbol __vmprof_eval_vmprof could not be found\n");
+            exit(-1);
+        }
+    }
+
+    return ptr == g_symbol;
+#endif
+}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -33,4 +33,7 @@
 RPY_EXTERN void vmprof_stack_free(void*);
 RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t);
 
+long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
+                                      intptr_t addr, int max_depth);
+
 #define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries)  (2 * (num_entries) + 4)
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -138,7 +138,7 @@
 #endif
 
 int vmp_walk_and_record_stack(PY_STACK_FRAME_T *frame, void ** result,
-                              int max_depth, intptr_t pc) {
+                              int max_depth, int signal, intptr_t pc) {
 
     // called in signal handler
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
@@ -158,7 +158,7 @@
         return 0;
     }
 
-    while (1) {
+    while (signal) {
         if (unw_is_signal_frame(&cursor)) {
             break;
         }
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.h b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.h
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
@@ -3,7 +3,7 @@
 #include "vmprof.h"
 
 int vmp_walk_and_record_stack(PY_STACK_FRAME_T * frame, void **data,
-                              int max_depth, intptr_t pc);
+                              int max_depth, int signal, intptr_t pc);
 
 int vmp_native_enabled(void);
 int vmp_native_enable(void);
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -44,19 +44,7 @@
 #define PY_THREAD_STATE_T void
 #define FRAME_STEP(f) f->next
 #define FRAME_CODE(f) f->
-
-#ifdef RPYTHON_LL2CTYPES
-#  define IS_VMPROF_EVAL(PTR) 0
-#else
-    // Is there is a way to tell the compiler
-    // that this prototype can have ANY return value. Just removing
-    // the return type will default to int
-    typedef long Signed;
-    RPY_EXTERN Signed __vmprof_eval_vmprof();
-#  define IS_VMPROF_EVAL(PTR) PTR == (void*)__vmprof_eval_vmprof
-#endif
-
-
+int IS_VMPROF_EVAL(void * ptr);
 #else
 #define RPY_EXTERN
 // for cpython
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
@@ -6,8 +6,8 @@
 
 #define MAX_INLINE_DEPTH  384
 
-static long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
-                                             intptr_t addr, int max_depth)
+long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
+                                      intptr_t addr, int max_depth)
 {
 #ifdef PYPY_JIT_CODEMAP
     void *codemap;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -101,7 +101,7 @@
     }
     frame = current->frame;
 #endif
-    return vmp_walk_and_record_stack(frame, result, max_depth, pc);
+    return vmp_walk_and_record_stack(frame, result, max_depth, 1, pc);
 }
 
 /* *************************************************************

From pypy.commits at gmail.com  Thu Feb  9 09:10:42 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 06:10:42 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: include to define some basics
 (such as intptr_t)
Message-ID: <589c7862.0fb81c0a.47e09.f2fb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90025:ac9fc28a42ac
Date: 2017-02-09 15:10 +0100
http://bitbucket.org/pypy/pypy/changeset/ac9fc28a42ac/

Log:	include to define some basics (such as intptr_t)

diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include "vmprof.h"
+
 void *pypy_find_codemap_at_addr(long addr, long *start_addr);
 long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
                                 long *current_pos_addr);

From pypy.commits at gmail.com  Thu Feb  9 09:44:15 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 09 Feb 2017 06:44:15 -0800 (PST)
Subject: [pypy-commit] stmgc default: some additions to the readme
Message-ID: <589c803f.8b811c0a.a5205.0184@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2009:e0d10ff9bbae
Date: 2017-02-09 15:44 +0100
http://bitbucket.org/pypy/stmgc/changeset/e0d10ff9bbae/

Log:	some additions to the readme

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -67,6 +67,7 @@
      - `forksupport`: support for forking an STM process
      - `extra`: on-commit and on-abort callback mechanism
      - `detach`: transaction detach mechanism (optimised transactional zones)
+     - `setup`: sets up the memory layout and segments
      
     Misc:
     
@@ -79,7 +80,6 @@
      - `misc`: mostly debug and testing interface
      - `pagecopy`: fast copy implementation for pages
      - `prebuilt`: logic for PyPy's prebuilt objects
-     - `setup`: setup code
        
 
 
@@ -88,7 +88,7 @@
 Tests are written in Python that calls the C-library through CFFI (Python package).
 
  1. install `pytest` and `cffi` packages for Python (via `pip`)
- 2. running py.test in c8/test should run all the tests (alternatively, the
+ 2. running `py.test` in `c8/test` should run all the tests (alternatively, the
     PyPy-checkout has a pytest.py script in its root-folder, which should work
     too)
 
@@ -99,7 +99,7 @@
 
  1. for example: `make build-demo_random2`
  2. then run `./build-demo_random2`
-
+ 
 
 ## Building PyPy-STM
 
@@ -159,6 +159,36 @@
  4. The script puts a `pypy-c` into `/pypy/goal/` that should be ready to run.
 
 
+### Log tools
 
+STMGC produces an event-log, if requested. Some tools to parse and analyse these
+logs are in the PyPy repository under `/pypy/stm/`. To produce a log, set the
+environment variable `PYPYSTM` to a file name. E.g.:
 
+`env PYPYSTM=log.pypystm time -p pypy-c program.py`
 
+and then see some statistics with 
+
+`/pypy/stm/print_stm_log.py log.pypystm`
+
+
+### Benchmarks
+
+In PyPy's benchmark repository (`https://bitbucket.org/pypy/benchmarks`) under
+`multithread` is a collection of multi-threaded Python programs to measure
+performance.
+
+One way to run them is to check out the branch `multithread-runner` and do the
+following:
+
+`./runner.py pypy-c config-raytrace.json result.json`
+
+This will use the configuration in the JSON file and run a few iterations; then
+write the result into a JSON file again. It will also print the command-line
+used to run the benchmark, in case you don't want to use the runner. The
+`getresults.py` script can be used to compare two versions of PyPy against each
+other, but it is very limited.
+
+
+
+

From pypy.commits at gmail.com  Thu Feb  9 10:03:03 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 07:03:03 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: move variable def out of for loop,
 rewrite IS_VMPROF_EVAL for pypy
Message-ID: <589c84a7.04abdf0a.22687.6d07@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90026:72660eb81a7b
Date: 2017-02-09 15:51 +0100
http://bitbucket.org/pypy/pypy/changeset/72660eb81a7b/

Log:	move variable def out of for loop, rewrite IS_VMPROF_EVAL for pypy

diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -18,22 +18,12 @@
 #endif
 
 
-void * g_symbol = 0;
-
+#ifdef RPYTHON_LL2CTYPES
+#define IS_VMPROF_EVAL(ptr) 0
+#else
+extern void * __vmprof_eval_vmprof;
 int IS_VMPROF_EVAL(void * ptr)
 {
-#ifdef RPYTHON_LL2CTYPES
-    return 0;
-#else
-
-    if (g_symbol == NULL) {
-        g_symbol = dlsym(RTLD_GLOBAL, "__vmprof_eval_vmprof");
-        if (g_symbol == NULL) {
-            fprintf(stderr, "symbol __vmprof_eval_vmprof could not be found\n");
-            exit(-1);
-        }
-    }
-
-    return ptr == g_symbol;
+    return ptr == __vmprof_eval_vmprof;
+}
 #endif
-}
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -390,10 +390,11 @@
             } case MARKER_STACKTRACE: {
                 long trace_count = _read_word(fileno);
                 long depth = _read_word(fileno);
+                long i;
 
                 LOG("stack 0x%llx %d %d\n", cur_pos, trace_count, depth);
 
-                for (long i = depth/2-1; i >= 0; i--) {
+                for (i = depth/2-1; i >= 0; i--) {
                     long kind = (long)_read_addr(fileno);
                     void * addr = _read_addr(fileno);
                     if (kind == VMPROF_NATIVE_TAG) {
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -105,6 +105,12 @@
         result[n++] = (void*)frame->value;
         *depth = n;
     }
+#ifdef PYPY_JIT_CODEMAP
+    else if (frame->kind == VMPROF_JITTED_TAG) {
+        intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
+        *depth = vmprof_write_header_for_jit_addr(result, *depth, pc, max_depth);
+    }
+#endif
 
 
 #endif
@@ -236,7 +242,7 @@
             //
 #ifdef PYPY_JIT_CODEMAP
             if (func_addr == 0 && top_most_frame->kind == VMPROF_JITTED_TAG) {
-                intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
+                intptr_t pc = ((intptr_t*)(top_most_frame->value - sizeof(intptr_t)))[0];
                 depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
                 frame = FRAME_STEP(frame);
             } else if (func_addr != 0x0) {
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -213,7 +213,6 @@
             symbols.append(tree.name)
             if len(tree.children) == 0:
                 return
-            assert len(tree.children) == 1
             for child in tree.children.values():
                 walk(child, symbols)
         symbols = []

From pypy.commits at gmail.com  Thu Feb  9 10:03:05 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 09 Feb 2017 07:03:05 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: exchange header,
 add paramter to _write_python_stack_entry
Message-ID: <589c84a9.c62e1c0a.63709.0bcb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90027:66868b7d5e79
Date: 2017-02-09 15:56 +0100
http://bitbucket.org/pypy/pypy/changeset/66868b7d5e79/

Log:	exchange header, add paramter to _write_python_stack_entry

diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -19,7 +19,7 @@
 
 
 #ifdef RPYTHON_LL2CTYPES
-#define IS_VMPROF_EVAL(ptr) 0
+int IS_VMPROF_EVAL(void * ptr) { return 0; }
 #else
 extern void * __vmprof_eval_vmprof;
 int IS_VMPROF_EVAL(void * ptr)
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -54,7 +54,7 @@
     return _vmp_profiles_lines;
 }
 
-static PY_STACK_FRAME_T * _write_python_stack_entry(PY_STACK_FRAME_T * frame, void ** result, int * depth)
+static PY_STACK_FRAME_T * _write_python_stack_entry(PY_STACK_FRAME_T * frame, void ** result, int * depth, int max_depth)
 {
     int len;
     int addr;
@@ -122,7 +122,7 @@
                                      int max_depth, int depth, intptr_t pc)
 {
     while (depth < max_depth && frame) {
-        frame = _write_python_stack_entry(frame, result, &depth);
+        frame = _write_python_stack_entry(frame, result, &depth, max_depth);
     }
     return depth;
 }
@@ -218,7 +218,7 @@
             {
 #endif
                 if (top_most_frame != NULL) {
-                    top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth);
+                    top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth, max_depth);
                 } else {
                     // Signals can occur at the two places (1) and (2), that will
                     // have added a stack entry, but the function __vmprof_eval_vmprof
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "vmprof.h"
+#include <stdint.h>
 
 void *pypy_find_codemap_at_addr(long addr, long *start_addr);
 long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,

From pypy.commits at gmail.com  Thu Feb  9 10:33:25 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 09 Feb 2017 07:33:25 -0800 (PST)
Subject: [pypy-commit] stmgc default: add link to pypy build instructions
Message-ID: <589c8bc5.4b371c0a.df433.1ddb@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2010:d22594d8ea5c
Date: 2017-02-09 16:33 +0100
http://bitbucket.org/pypy/stmgc/changeset/d22594d8ea5c/

Log:	add link to pypy build instructions

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -97,7 +97,8 @@
 Demos are small C programs that use the STMGC library directly. They sometimes
 expose real data-races that the sequential Python tests cannot expose.
 
- 1. for example: `make build-demo_random2`
+ 1. for 
+ example: `make build-demo_random2`
  2. then run `./build-demo_random2`
  
 
@@ -107,6 +108,9 @@
 STMGC, run the `import_stmgc.py` script in `/rpython/translator/stm/`. In the
 following, `/` is the root of your PyPy checkout.
 
+ 0. Follow the [http://doc.pypy.org/en/latest/build.html](build instructions)
+    for PyPy until you get to the point to run the translation.
+
  1. The Makefile expects a `gcc-seg-gs` executable to be on the `$PATH`. This
     should be a GCC that is either patched or a wrapper to GCC 6.1 that passes
     the necessary options. In my case, this is a script that points to my custom
@@ -130,7 +134,7 @@
         STM=--stm #--stm
         JIT=-Ojit #-Ojit #-O2
         VERSION=$(hg id -i)
-        time  ionice -c3 pypy ~/pypy_dir/rpython/bin/rpython --no-shared --source $STM $JIT  targetpypystandalone.py
+        ionice -c3 pypy ~/pypy_dir/rpython/bin/rpython --no-shared --source $STM $JIT  targetpypystandalone.py
         # --no-allworkingmodules
         
         notify-send "PyPy" "C source generated."
@@ -165,7 +169,7 @@
 logs are in the PyPy repository under `/pypy/stm/`. To produce a log, set the
 environment variable `PYPYSTM` to a file name. E.g.:
 
-`env PYPYSTM=log.pypystm time -p pypy-c program.py`
+`env PYPYSTM=log.pypystm pypy-c program.py`
 
 and then see some statistics with 
 

From pypy.commits at gmail.com  Thu Feb  9 10:34:23 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 09 Feb 2017 07:34:23 -0800 (PST)
Subject: [pypy-commit] stmgc default: fix
Message-ID: <589c8bff.499adf0a.bfb67.7e5f@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2011:048795097bf2
Date: 2017-02-09 16:34 +0100
http://bitbucket.org/pypy/stmgc/changeset/048795097bf2/

Log:	fix

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@
 STMGC, run the `import_stmgc.py` script in `/rpython/translator/stm/`. In the
 following, `/` is the root of your PyPy checkout.
 
- 0. Follow the [http://doc.pypy.org/en/latest/build.html](build instructions)
+ 0. Follow the [build instructions](http://doc.pypy.org/en/latest/build.html)
     for PyPy until you get to the point to run the translation.
 
  1. The Makefile expects a `gcc-seg-gs` executable to be on the `$PATH`. This

From pypy.commits at gmail.com  Thu Feb  9 11:24:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Feb 2017 08:24:01 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: in-progress
Message-ID: <589c97a1.cd2f1c0a.c2729.2847@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90028:9242b40aa211
Date: 2017-02-09 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/9242b40aa211/

Log:	in-progress

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -269,6 +269,11 @@
         guardtok.faildescr.rd_locs = positions
         return faildescrindex, target
 
+    def get_target_for_failure_recovery_of_guard_compat(self):
+        exc = False
+        withfloats = True
+        return self.failure_recovery_code[exc + 2 * withfloats]
+
     def enter_portal_frame(self, op):
         if self.cpu.HAS_CODEMAP:
             self.codemap_builder.enter_portal_frame(op.getarg(0).getint(),
diff --git a/rpython/jit/backend/llsupport/guard_compat.py b/rpython/jit/backend/llsupport/guard_compat.py
--- a/rpython/jit/backend/llsupport/guard_compat.py
+++ b/rpython/jit/backend/llsupport/guard_compat.py
@@ -18,14 +18,18 @@
                              ('asmaddr', lltype.Signed))
 BACKEND_CHOICES = lltype.GcStruct('BACKEND_CHOICES',
                         ('bc_faildescr', llmemory.GCREF),
+                        ('bc_gcmap', lltype.Ptr(jitframe.GCMAP)),
                         ('bc_gc_table_tracer', llmemory.GCREF),
+                        ('bc_search_tree', lltype.Signed),
                         ('bc_most_recent', PAIR),
                         ('bc_list', lltype.Array(PAIR)))
 
 def _getofs(name):
     return llmemory.offsetof(BACKEND_CHOICES, name)
 BCFAILDESCR = _getofs('bc_faildescr')
+BCGCMAP = _getofs('bc_gcmap')
 BCGCTABLETRACER = _getofs('bc_gc_table_tracer')
+BCSEARCHTREE = _getofs('bc_search_tree')
 BCMOSTRECENT = _getofs('bc_most_recent')
 BCLIST = _getofs('bc_list')
 del _getofs
@@ -175,12 +179,9 @@
     bchoices = lltype.malloc(BACKEND_CHOICES, 1)
     bchoices.bc_faildescr = cast_instance_to_gcref(guard_compat_descr)
     bchoices.bc_gc_table_tracer = lltype.nullptr(llmemory.GCREF.TO)   # (*)
-    bchoices.bc_most_recent.gcref = gcref_to_unsigned(initial_gcref)
-    bchoices.bc_most_recent.asmaddr = -43  # (*)
-    bchoices.bc_list[0].gcref = gcref_to_unsigned(initial_gcref)
-    bchoices.bc_list[0].asmaddr = -43  # (*)
+    bchoices.bc_most_recent.gcref = r_uint(-1)
+    bchoices.bc_list[0].gcref = r_uint(-1)
     llop.gc_writebarrier(lltype.Void, bchoices)
-    # entries with (*) are fixed in patch_guard_compatible()
     return bchoices
 
 def descr_to_bchoices(descr):
@@ -191,32 +192,31 @@
     # ---no GC operation end---
     return bchoices
 
-def patch_guard_compatible(guard_token, rawstart, get_addr_in_gc_table,
-                           gc_table_tracer):
-    # go to the address in the gctable, number 'bindex'
-    bindex = guard_token.guard_compat_bindex
-    choices_addr = get_addr_in_gc_table(bindex)
-    sequel_label = rawstart + guard_token.pos_jump_offset
-    failure_recovery = rawstart + guard_token.pos_recovery_stub
-    gcmap = guard_token.gcmap
-    # choices_addr:     points to bchoices in the GC table
-    # sequel_label:     "sequel:" label above
-    # failure_recovery: failure recovery address
+def patch_guard_compatible(guard_token, get_addr_in_gc_table,
+                           gc_table_tracer, search_tree_addr):
     guard_compat_descr = guard_token.faildescr
     assert isinstance(guard_compat_descr, GuardCompatibleDescr)
+    #
+    # read the initial value of '_backend_choices_addr', which is used
+    # to store the index of the '_backend_choices' gc object in the gc
+    # table
+    bindex = guard_compat_descr._backend_choices_addr
+    #
+    # go to this address in the gctable
+    choices_addr = get_addr_in_gc_table(bindex)
+    #
+    # now fix '_backend_choices_addr' to really point to the raw address
+    # in the gc table
     guard_compat_descr._backend_choices_addr = choices_addr
-    guard_compat_descr._backend_sequel_label = sequel_label
-    guard_compat_descr._backend_failure_recovery = failure_recovery
-    guard_compat_descr._backend_gcmap = gcmap
     #
     bchoices = descr_to_bchoices(guard_compat_descr)
     assert len(bchoices.bc_list) == 1
     assert (cast_gcref_to_instance(GuardCompatibleDescr, bchoices.bc_faildescr)
             is guard_compat_descr)
+    bchoices.bc_gcmap = guard_token.gcmap
     bchoices.bc_gc_table_tracer = lltype.cast_opaque_ptr(llmemory.GCREF,
                                                          gc_table_tracer)
-    bchoices.bc_most_recent.asmaddr = sequel_label
-    bchoices.bc_list[0].asmaddr = sequel_label
+    bchoices.bc_search_tree = search_tree_addr
 
 def invalidate_pair(bchoices, pair_ofs):
     gcref_base = lltype.cast_opaque_ptr(llmemory.GCREF, bchoices)
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -364,8 +364,7 @@
                 self.handle_call_assembler(op)
                 continue
             if op.getopnum() == rop.GUARD_COMPATIBLE:
-                self.handle_guard_compatible(op)
-                continue
+                self.prepare_guard_compatible(op)
             if op.getopnum() == rop.JUMP or op.getopnum() == rop.FINISH:
                 self.emit_pending_zeros()
             #
@@ -978,8 +977,10 @@
             self.gcrefs_recently_loaded[index] = load_op
         return load_op
 
-    def handle_guard_compatible(self, op):
+    def prepare_guard_compatible(self, op):
         from rpython.jit.backend.llsupport import guard_compat
+        # don't use _gcref_index here: we need our own index for
+        # the _backend_choices object
         c = op.getarg(1)
         assert isinstance(c, ConstPtr)
         descr = op.getdescr()
@@ -987,9 +988,8 @@
         bcindex = len(self.gcrefs_output_list)
         gcref = lltype.cast_opaque_ptr(llmemory.GCREF, bchoices)
         self.gcrefs_output_list.append(gcref)
-        new_op = op.copy_and_change(rop.GUARD_COMPATIBLE,
-                                    [op.getarg(0), ConstInt(bcindex)])
-        self.emit_op(new_op)
+        assert isinstance(descr, guard_compat.GuardCompatibleDescr)
+        descr._backend_choices_addr = bcindex  # fixed in patch_guard_compatible
 
 @always_inline
 def cpu_simplify_scale(cpu, index_box, factor, offset):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -782,9 +782,9 @@
             addr = rawstart + tok.pos_jump_offset
             tok.faildescr.adr_jump_offset = addr
             if tok.guard_compatible():
-                guard_compat.patch_guard_compatible(tok, rawstart,
-                                                    self._addr_from_gc_table,
-                                                    self.gc_table_tracer)
+                guard_compat.patch_guard_compatible(
+                    tok, self._addr_from_gc_table,
+                    self.gc_table_tracer, self.guard_compat_search_tree)
                 continue
             descr = tok.faildescr
             if descr.loop_version():
@@ -1832,18 +1832,11 @@
             self.mc.UCOMISD(locs[0], locs[1])
         else:
             self.mc.CMP(locs[0], locs[1])
+        guard_token._guard_value_on = locs[0].value
         self.guard_success_cc = rx86.Conditions['E']
         self.implement_guard(guard_token)
 
-    def genop_guard_guard_compatible(self, guard_op, guard_token, locs, ign):
-        loc_reg, loc_imm, loc_reg2 = locs
-        assert isinstance(loc_reg, RegLoc)
-        assert isinstance(loc_imm, ImmedLoc)    # index of 'backend_choices'
-        assert isinstance(loc_reg2, RegLoc)
-        self.load_reg_from_gc_table(loc_reg2.value, loc_imm.value)
-        guard_compat.generate_guard_compatible(self, guard_token,
-                                               loc_reg.value, loc_imm.value,
-                                               loc_reg2.value)
+    genop_guard_guard_compatible = genop_guard_guard_value
 
     def _cmp_guard_class(self, locs):
         loc_ptr = locs[0]
@@ -1982,6 +1975,11 @@
                              guardtok.faildescr, regalloc)
         #
         faildescrindex, target = self.store_info_on_descr(startpos, guardtok)
+        if guardtok.guard_compatible():
+            assert startpos == self.mc.get_relative_pos()
+            guard_compat.generate_recovery_stub(self, guardtok)
+            xXXXx
+        #
         self.push_from_gc_table(faildescrindex)
         self.push_gcmap(self.mc, guardtok.gcmap, push=True)
         self.mc.JMP(imm(target))
diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py
--- a/rpython/jit/backend/x86/guard_compat.py
+++ b/rpython/jit/backend/x86/guard_compat.py
@@ -1,5 +1,5 @@
 from rpython.rtyper.annlowlevel import llhelper
-from rpython.jit.backend.x86 import rx86, codebuf, regloc
+from rpython.jit.backend.x86 import rx86, codebuf, regloc, callbuilder
 from rpython.jit.backend.x86.regalloc import gpr_reg_mgr_cls
 from rpython.jit.backend.x86.arch import WORD, IS_X86_64, IS_X86_32
 from rpython.jit.backend.x86.arch import DEFAULT_FRAME_BYTES
@@ -17,27 +17,8 @@
 #     CMP reg, reg2
 #     JNE recovery_stub
 #   sequel:
-#     <reg2 not used any more>
 #
-# The difference is that 'recovery_stub' does not jump to one of the
-# 'failure_recovery_code' versions, but instead it jumps to
-# 'expand_guard_compatible'.  The latter calls invoke_find_compatible.
-# The result is one of:
-#
-#   * 0: bail out.  We jump to the 'failure_recovery_code'.
-#
-#   * -1: continue running on the same path.  We patch ofs(const-ptr)
-#     to contain the new value, and jump to 'sequel'.
-#
-#   * otherwise, it's the address of a bridge.  We jump to that bridge.
-#
-# This is the basic idea, but not the truth.  Things are more
-# complicated because we cache in the assembler the
-# invoke_find_compatible call results.  'expand_guard_compatible'
-# actually allocates a '_backend_choices' object, copies on it
-# various data it got from the recovery_stub, then patches the
-# recovery stub to this (the original recovery stub was padded if
-# necessary to have enough room):
+# The difference is in the 'recovery_stub':
 #
 #   recovery_stub:
 #     MOV R11, [RIP + ofs(_backend_choices)]
@@ -52,9 +33,8 @@
 # The faildescr for the GUARD_COMPATIBLE is a GuardCompatibleDescr.
 # Fields relevant for this discussion:
 #
-#     - _backend_ptr_addr: points inside the GC table, to ofs(const-ptr).
-#                          ofs(_backend_choices) is just afterwards.
-#                          Initially _backend_choices is NULL.
+#     - _backend_choices_addr: points inside the GC table, to
+#                              ofs(_backend_choices)
 #     - adr_jump_offset: raw address of the 'sequel' label (this field
 #                        is the same as on any other GuardDescr)
 #
@@ -94,8 +74,8 @@
 # When find_compatible() returns 0, it is not stored in bc_list,
 # but still stored in bc_most_recent, with 'guard_compat_recovery'
 # as the 'asmaddr'.  Here is 'guard_compat_recovery': it emulates
-# generate_quick_failure() from assembler.py, and so it plays the role
-# of the original (patched) recovery stub.
+# the non-GUARD_COMPATIBLE case of generate_quick_failure() from
+# assembler.py.
 #
 #   guard_compat_recovery:
 #     PUSH R11
@@ -105,8 +85,8 @@
 # Here is the x86-64 runtime code to walk the tree:
 #
 #   search_tree:
-#     MOV [ESP+8], RCX                     # save the original value
-#     MOV [ESP+16], R11                    # save the _backend_choices object
+#     MOV [RSP+8], RCX                     # save the original value
+#     MOV [RSP+16], R11                    # save the _backend_choices object
 #     MOV RCX, [R11 + bc_list.length]      # a power of two minus one
 #     ADD R11, $bc_list.items
 #     JMP loop
@@ -126,22 +106,24 @@
 #
 #   found:
 #     MOV R11, [R11 + 8*RCX]             # address to jump to next
-#     MOV RCX, [ESP+16]                  # reload the _backend_choices object
+#     MOV RCX, [RSP+16]                  # reload the _backend_choices object
 #     MOV [RCX + bc_most_recent], RAX
 #     MOV [RCX + bc_most_recent + 8], R11
-#     MOV RCX, [ESP+8]                   # restore saved value
+#     MOV RCX, [RSP+8]                   # restore saved value
 #     POP RAX                            # pushed by the caller
-#     JMP *R11
+#     JMP *R11                           # can't jump to guard_compat_recovery
 #
 #   not_found:
 #     <save all registers to the jitframe RBP,
 #         reading and popping the original RAX and RCX off the stack>
-#     <call invoke_find_compatible(_backend_choices=[RSP], value=RAX),
-#                                  jitframe=RBP>
+#     <build an array of two words on the stack, with _backend_choices
+#         and value; the 'value' will be overwritten by
+#         invoke_find_compatible with the address to jump to next>
+#     <call invoke_find_compatible(p_arg=RSP, jitframe=RBP>
 #     <_reload_frame_if_necessary>
-#     MOV R11, RAX
 #     <restore all registers>
-#     JMP *R11
+#     MOV R11, [RSP+array_element_1]     # reload the _backend_choices object
+#     JMP *[RSP+array_element_2]         # may jump to guard_compat_recovery
 #
 #
 # invoke_find_compatible(bchoices, new_gcref, jitframe):
@@ -168,7 +150,7 @@
 # Other issues: compile_bridge() called on a GuardCompatibleDescr must
 # not to do any patching, but instead it needs to clear
 # bchoices.bc_most_recent.  Otherwise, we will likely directly jump to
-# <failure_recovery> next time, if the newly added gcref is still in
+# <guard_compat_recovery> next time, if the newly added gcref is still in
 # bc_most_recent.gcref.  (We can't add it to bc_most_recent or bc_list
 # from compile_bridge(), because we don't know what the gcref should
 # be, but it doesn't matter.)
@@ -186,29 +168,36 @@
     mc.overwrite(jmp_location-1, chr(offset))
 
 def build_once(assembler):
+    build_once_search_tree(assembler)
+    build_once_guard_compat_recovery(assembler)
+
+def build_once_search_tree(assembler):
     """Generate the 'search_tree' block of code"""
     rax = regloc.eax.value
-    rdx = regloc.edx.value
+    rcx = regloc.ecx.value
     rdi = regloc.edi.value
     r11 = regloc.r11.value
-    frame_size = DEFAULT_FRAME_BYTES + 2 * WORD
-    # contains two extra words on the stack:
-    #    - saved RDX
+    frame_size = DEFAULT_FRAME_BYTES + 1 * WORD
+    # contains one extra word on the stack:
     #    - saved RAX
 
     mc = codebuf.MachineCodeBlockWrapper()
     mc.force_frame_size(frame_size)
+    mc.INT3()
     if IS_X86_32:    # save edi as an extra scratch register
+        XXX
         mc.MOV_sr(3*WORD, rdi)
         r11 = rdi    # r11 doesn't exist on 32-bit, use "edi" instead
 
+    mc.MOV_sr(1*WORD, rcx)                  # MOV [RSP+8], ECX
+    mc.MOV_sr(2*WORD, r11)                  # MOV [RSP+16], R11
+
     ofs1 = _real_number(BCLIST + BCLISTLENGTHOFS)
     ofs2 = _real_number(BCLIST + BCLISTITEMSOFS)
-    mc.MOV_sr(2*WORD, rdx)                  # MOV [RSP+16], RDX
-    mc.MOV_rm(r11, (rdx, ofs1))             # MOV R11, [RDX + bc_list.length]
-    # in the sequel, "RDX + bc_list.items" is a pointer to the leftmost
+    mc.MOV_rm(rcx, (r11, ofs1))             # MOV RCX, [R11 + bc_list.length]
+    # in the sequel, "R11 + bc_list.items" is a pointer to the leftmost
     # array item of the range still under consideration.  The length of
-    # this range is R11, which is always a power-of-two-minus-1.
+    # this range is RCX, which is always a power-of-two-minus-1.
     mc.JMP_l8(0)                            # JMP loop
     jmp_location = mc.get_relative_pos()
     mc.force_frame_size(frame_size)
@@ -216,28 +205,29 @@
     SH = 3 if IS_X86_64 else 2
 
     right_label = mc.get_relative_pos()
-    mc.LEA_ra(rdx, (rdx, r11, SH, WORD))    # LEA RDX, [RDX + 8*R11 + 8]
+    mc.LEA_ra(r11, (r11, rcx, SH, WORD))    # LEA R11, [R11 + 8*RCX + 8]
     left_label = mc.get_relative_pos()
-    mc.SHR_ri(r11, 1)                       # SHR R11, 1
+    mc.SHR_ri(rcx, 1)                       # SHR RCX, 1
     mc.J_il8(rx86.Conditions['Z'], 0)       # JZ not_found
     jz_location = mc.get_relative_pos()
 
     _fix_forward_label(mc, jmp_location)    # loop:
-    mc.CMP_ra(rax, (rdx, r11, SH, ofs2-WORD))
-                                            # CMP RAX, [RDX + items + 8*R11 - 8]
+    mc.CMP_ra(rax, (r11, rcx, SH, ofs2-WORD))
+                                            # CMP RAX, [R11 + items + 8*RCX - 8]
     mc.J_il8(rx86.Conditions['A'], right_label - (mc.get_relative_pos() + 2))
     mc.J_il8(rx86.Conditions['NE'], left_label - (mc.get_relative_pos() + 2))
 
-    mc.MOV_ra(r11, (rdx, r11, SH, ofs2))    # MOV R11, [RDX + items + 8*R11]
-    mc.MOV_rs(rdx, 2*WORD)                  # MOV RDX, [RSP+16]
+    mc.MOV_ra(r11, (r11, rcx, SH, ofs2))    # MOV R11, [R11 + items + 8*RCX]
+    mc.MOV_rs(rcx, 2*WORD)                  # MOV RCX, [RSP+16]
     ofs = _real_number(BCMOSTRECENT)
-    mc.MOV_mr((rdx, ofs), rax)              # MOV [RDX+bc_most_recent], RAX
-    mc.MOV_mr((rdx, ofs+WORD), r11)         # MOV [RDX+bc_most_recent+8], R11
+    mc.MOV_mr((rcx, ofs), rax)              # MOV [RCX+bc_most_recent], RAX
+    mc.MOV_mr((rcx, ofs+WORD), r11)         # MOV [RCX+bc_most_recent+8], R11
+    mc.MOV_rs(rcx, 1*WORD)                  # MOV RCX, [RSP+8]
     mc.POP_r(rax)                           # POP RAX
-    mc.POP_r(rdx)                           # POP RDX
     if IS_X86_64:
         mc.JMP_r(r11)                       # JMP *R11
     elif IS_X86_32:
+        XXX
         mc.MOV_sr(0, r11) # r11==rdi here
         mc.MOV_rs(rdi, WORD)
         mc.JMP_s(0)
@@ -246,24 +236,28 @@
     _fix_forward_label(mc, jz_location)     # not_found:
 
     if IS_X86_32:
+        XXX
         mc.MOV_rs(rdi, 3*WORD)
 
-    # read and pop the original RAX and RDX off the stack
-    base_ofs = assembler.cpu.get_baseofs_of_frame_field()
-    v = gpr_reg_mgr_cls.all_reg_indexes[rax]
-    mc.POP_b(v * WORD + base_ofs)           # POP [RBP + saved_rax]
-    v = gpr_reg_mgr_cls.all_reg_indexes[rdx]
-    mc.POP_b(v * WORD + base_ofs)           # POP [RBP + saved_rdx]
-    # save all other registers to the jitframe RBP
-    assembler._push_all_regs_to_frame(mc, [regloc.eax, regloc.edx],
-                                      withfloats=True)
+    # The _backend_choices object is still referenced from [RSP+16]
+    # (which becomes [RSP+8] after the POP), where it is the first of a
+    # two-words array passed as argument to invoke_find_compatible().
+    # The second word is the value, from RAX, which we store now.
+    mc.MOV_sr(3*WORD, rax)                  # MOV [RSP+24], RAX
+
+    # restore RAX and RCX
+    mc.MOV_rs(rcx, 1*WORD)                  # MOV RCX, [RSP+8]
+    mc.POP_r(rax)                           # POP RAX
+
+    # save all registers to the jitframe RBP
+    assembler._push_all_regs_to_frame(mc, [], withfloats=True)
 
     if IS_X86_64:
-        mc.MOV_rs(rdi, 0)                   # MOV RDI, [RSP]
-        mc.MOV_rr(regloc.esi.value, rax)    # MOV RSI, RAX
-        mc.MOV_rr(regloc.edx.value,         # MOV RDX, RBP
+        mc.LEA_rs(rdi, 2 * WORD)            # LEA RDI, [RSP+8]
+        mc.MOV_rr(regloc.esi.value,         # MOV RSI, RBP
                   regloc.ebp.value)
     elif IS_X86_32:
+        XXX
         # argument #1 is already in [ESP]
         mc.MOV_sr(1 * WORD, rax)
         mc.MOV_sr(2 * WORD, regloc.ebp.value)
@@ -273,58 +267,89 @@
     llfunc = assembler.cpu.cast_ptr_to_int(llfunc)
     mc.CALL(regloc.imm(llfunc))             # CALL invoke_find_compatible
     assembler._reload_frame_if_necessary(mc)
-    if IS_X86_64:
-        mc.MOV_rr(r11, rax)                 # MOV R11, RAX
-    elif IS_X86_32:
-        mc.MOV_sr(0, rax)
 
     # restore the registers that the CALL has clobbered, plus the ones
     # containing GC pointers that may have moved.  That means we just
-    # restore them all.  (We restore RAX and RDX and RDI too.)
+    # restore them all.
     assembler._pop_all_regs_from_frame(mc, [], withfloats=True)
+
+    # jump to 'array_element_2'.  In case this goes to
+    # guard_compat_recovery, we also reload the _backend_choices
+    # object from 'array_element_1' (the GC may have moved it, or
+    # it may be a completely new object).
     if IS_X86_64:
-        mc.JMP_r(r11)                       # JMP *R11
+        mc.MOV_rs(r11, 1*WORD)              # MOV R11, [RSP+8]
+        mc.JMP_s(2*WORD)                    # JMP *[RSP+16]
     elif IS_X86_32:
+        XXX
         mc.JMP_s(0)
 
     assembler.guard_compat_search_tree = mc.materialize(assembler.cpu, [])
 
 
-def generate_guard_compatible(assembler, guard_token, reg, bindex, reg2):
-    mc = assembler.mc
-    rax = regloc.eax.value
-    rdx = regloc.edx.value
+def build_once_guard_compat_recovery(assembler):
+    """Generate the 'guard_compat_recovery' block of code"""
+    r11 = regloc.r11.value
+    mc = codebuf.MachineCodeBlockWrapper()
+
+    ofs = _real_number(BCGCMAP)
+    mc.PUSH_r(r11)
+    mc.PUSH_m((r11, ofs))
+    target = assembler.get_target_for_failure_recovery_of_guard_compat()
+    mc.JMP(regloc.imm(target))
+
+    assembler.guard_compat_recovery = mc.materialize(assembler.cpu, [])
+
+
+def generate_recovery_stub(assembler, guard_token):
+    r11 = regloc.r11.value
     frame_size = DEFAULT_FRAME_BYTES
 
+    descr = guard_token.faildescr
+    assert isinstance(descr, GuardCompatibleDescr)
+    assembler.load_reg_from_gc_table(r11, descr._backend_choices_addr)
+
+    mc = assembler.mc
+    reg = guard_token._guard_value_on
     ofs = _real_number(BCMOSTRECENT)
-    mc.CMP_rm(reg, (reg2, ofs))             # CMP reg, [reg2 + bc_most_recent]
+    mc.CMP_rm(reg, (r11, ofs))              # CMP reg, [R11 + bc_most_recent]
     mc.J_il8(rx86.Conditions['NE'], 0)      # JNE slow_case
     jne_location = mc.get_relative_pos()
 
-    mc.JMP_m((reg2, ofs + WORD))            # JMP *[reg2 + bc_most_recent + 8]
+    mc.JMP_m((r11, ofs + WORD))             # JMP *[R11 + bc_most_recent + 8]
     mc.force_frame_size(frame_size)
 
     _fix_forward_label(mc, jne_location)    # slow_case:
-    mc.PUSH_r(rdx)                          # PUSH RDX
     mc.PUSH_r(rax)                          # PUSH RAX
-    # manually move reg to RAX and reg2 to RDX
-    if reg2 == rax:
-        if reg == rdx:
-            mc.XCHG_rr(rax, rdx)
-            reg = rax
-        else:
-            mc.MOV_rr(rdx, rax)
-        reg2 = rdx
     if reg != rax:
-        assert reg2 != rax
-        mc.MOV_rr(rax, reg)
-    if reg2 != rdx:
-        mc.MOV_rr(rdx, reg2)
+        mc.MOV_rr(rax, reg)                 # MOV RAX, reg
 
-    mc.JMP(regloc.imm(assembler.guard_compat_search_tree))
-    mc.force_frame_size(frame_size)
+    ofs = _real_number(BCSEARCHTREE)
+    mc.JMP_m((r11, ofs))                    # JMP *[R11 + bc_search_tree]
 
-    # abuse this field to store the 'sequel' relative offset
-    guard_token.pos_jump_offset = mc.get_relative_pos()
-    guard_token.guard_compat_bindex = bindex
-    assembler.pending_guard_tokens.append(guard_token)
+
+#def generate_guard_compatible(assembler, guard_token, reg, reg2, gctable_index):
+#    mc = assembler.mc
+#    mc.CMP_rr(reg, reg2)                    # CMP reg, reg2
+#    mc.J_il8(rx86.Conditions['E'], 0)       # JE sequel
+#    je_location = mc.get_relative_pos()
+#
+#    self.push_from_gc_table(guard_token.faildescrindex)
+#    mc.JMP(regloc.imm(assembler.guard_compat_second_case))
+#
+#    padding_end = start_pos + size_general_case - 2
+#    while mc.get_relative_pos() < padding_end:
+#        mc.INT3()
+#
+#    padding_end = mc.get_relative_pos()    # in case it is actually bigger
+#    block_size = padding_end - start_pos + 2
+#    assert 0 < block_size <= 255
+#    mc.writechar(chr(block_size))
+#    assert 0 <= reg <= 15 and 0 <= reg2 <= 15
+#    mc.writechar(chr((reg2 << 4) | reg))
+#
+#    # abuse this field to store the 'sequel' relative offset
+#    guard_token.pos_jump_offset = mc.get_relative_pos()
+#    guard_token.guard_compat_bindex = gctable_index
+#    guard_token..............
+#    assembler.pending_guard_tokens.append(guard_token)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -478,17 +478,7 @@
         y = self.loc(op.getarg(1))
         self.perform_guard(op, [x, y], None)
 
-    def consider_guard_compatible(self, op):
-        op.getdescr().make_a_counter_per_value(op, -1)   # -1 not used here
-        args = op.getarglist()
-        assert args[0].type == REF             # only supported case for now
-        assert isinstance(args[1], ConstInt)   # by rewrite.py
-        tmp_box = TempVar()
-        x = self.rm.make_sure_var_in_reg(args[0])
-        y = self.loc(args[1])
-        z = self.rm.force_allocate_reg(tmp_box, args)
-        self.rm.possibly_free_var(tmp_box)
-        self.perform_guard(op, [x, y, z], None)
+    consider_guard_compatible = consider_guard_value
 
     def consider_guard_class(self, op):
         assert not isinstance(op.getarg(0), Const)
diff --git a/rpython/jit/backend/x86/test/test_compatible.py b/rpython/jit/backend/x86/test/test_compatible.py
--- a/rpython/jit/backend/x86/test/test_compatible.py
+++ b/rpython/jit/backend/x86/test/test_compatible.py
@@ -19,39 +19,42 @@
         mc.writechar('\x00')   # 4 gctable entries; 'bchoices' will be #3
     #
     if IS_X86_64:
-        mc.MOV(regloc.ecx, regloc.edx)
-        mc.MOV(regloc.edx, regloc.edi)
-        mc.MOV(regloc.eax, regloc.esi)
+        mc.MOV(regloc.ecx, regloc.edx)    # jitframe
+        mc.MOV(regloc.r11, regloc.edi)    # _backend_choices
+        mc.MOV(regloc.eax, regloc.esi)    # guarded value
     elif IS_X86_32:
+        XXX
         mc.MOV_rs(regloc.edx.value, 4)
         mc.MOV_rs(regloc.eax.value, 8)
         mc.MOV_rs(regloc.ecx.value, 12)
     #
     mc.PUSH(regloc.ebp)
-    mc.SUB(regloc.esp, regloc.imm(148 - 2*WORD)) # make a frame, and align stack
+    mc.SUB(regloc.esp, regloc.imm(144 - 2*WORD)) # make a frame, and align stack
     mc.MOV(regloc.ebp, regloc.ecx)
     #
-    mc.PUSH(regloc.imm(0xdddd))
     mc.PUSH(regloc.imm(0xaaaa))
-    mc.JMP(regloc.imm(cpu.assembler.guard_compat_search_tree))
+    # jump to guard_compat_search_tree, but carefully: don't overwrite R11
+    mc.MOV(regloc.esi, regloc.imm(cpu.assembler.guard_compat_search_tree))
+    mc.JMP_r(regloc.esi.value)
     sequel = mc.get_relative_pos()
     #
-    mc.force_frame_size(148)
+    mc.force_frame_size(144)
     mc.SUB(regloc.eax, regloc.edx)
-    mc.ADD(regloc.esp, regloc.imm(148 - 2*WORD))
+    mc.ADD(regloc.esp, regloc.imm(144 - 2*WORD))
     mc.POP(regloc.ebp)
     mc.RET()
     #
     extra_paths = []
     for i in range(11):
-        mc.force_frame_size(148)
+        mc.force_frame_size(144)
         extra_paths.append(mc.get_relative_pos())
         mc.MOV(regloc.eax, regloc.imm(1000000 + i))
-        mc.ADD(regloc.esp, regloc.imm(148 - 2*WORD))
+        mc.ADD(regloc.esp, regloc.imm(144 - 2*WORD))
         mc.POP(regloc.ebp)
         mc.RET()
     failure = extra_paths[10]
     rawstart = mc.materialize(cpu, [])
+    print 'rawstart:', hex(rawstart)
     call_me = rffi.cast(lltype.Ptr(lltype.FuncType(
         [lltype.Ptr(BACKEND_CHOICES), llmemory.GCREF,
          lltype.Ptr(jitframe.JITFRAME)], lltype.Signed)),
@@ -63,16 +66,17 @@
     llop.raw_store(lltype.Void, rawstart, 3 * WORD, bchoices)
 
     class FakeGuardToken:
-        guard_compat_bindex = 3
-        pos_jump_offset = sequel
-        pos_recovery_stub = failure
+        #pos_jump_offset = sequel
+        #pos_recovery_stub = failure
         gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), 0x10111213)
         faildescr = guard_compat_descr
     guard_token = FakeGuardToken()
+    guard_compat_descr._backend_choices_addr = 3
 
-    patch_guard_compatible(guard_token, rawstart,
+    patch_guard_compatible(guard_token,
                            lambda index: rawstart + index * WORD,
-                           lltype.nullptr(llmemory.GCREF.TO))
+                           lltype.nullptr(llmemory.GCREF.TO),
+                           9999)
 
     # ---- ready ----
 

From pypy.commits at gmail.com  Thu Feb  9 11:45:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Feb 2017 08:45:53 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: in-progress
Message-ID: <589c9cc1.958c1c0a.54147.37c5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90029:c0a1749a2014
Date: 2017-02-09 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/c0a1749a2014/

Log:	in-progress

diff --git a/rpython/jit/backend/llsupport/guard_compat.py b/rpython/jit/backend/llsupport/guard_compat.py
--- a/rpython/jit/backend/llsupport/guard_compat.py
+++ b/rpython/jit/backend/llsupport/guard_compat.py
@@ -97,24 +97,29 @@
     return rffi.cast(lltype.Unsigned, gcref)
 
 
+P_ARG = lltype.Struct('P_ARG', ('new_gcref', llmemory.GCREF),
+                               ('bchoices', lltype.Ptr(BACKEND_CHOICES)),
+                               ('jump_to', lltype.Signed))
+
 INVOKE_FIND_COMPATIBLE_FUNC = lltype.Ptr(lltype.FuncType(
-                [lltype.Ptr(BACKEND_CHOICES), llmemory.GCREF,
-                 lltype.Ptr(jitframe.JITFRAME)],
+                [lltype.Ptr(P_ARG), lltype.Ptr(jitframe.JITFRAME)],
                 lltype.Signed))
 
 @specialize.memo()
 def make_invoke_find_compatible(cpu):
-    def invoke_find_compatible(bchoices, new_gcref, jitframe):
+    def invoke_find_compatible(p_arg, jitframe):
+        bchoices = p_arg.bchoices
+        new_gcref = p_arg.new_gcref
         descr = bchoices.bc_faildescr
         descr = cast_gcref_to_instance(GuardCompatibleDescr, descr)
         try:
-            jitframe.jf_gcmap = descr._backend_gcmap
+            jitframe.jf_gcmap = bchoices.bc_gcmap
             result = descr.find_compatible(cpu, new_gcref)
             if result == 0:
-                result = descr._backend_failure_recovery
+                result = cpu.assembler.guard_compat_recovery
             else:
                 if result == -1:
-                    result = descr._backend_sequel_label
+                    result = descr.adr_jump_offset
                 bchoices = add_in_tree(bchoices, new_gcref, result)
                 # ---no GC operation---
                 choices_addr = descr._backend_choices_addr  # GC table
@@ -129,9 +134,10 @@
             if not we_are_translated():
                 import sys, pdb
                 pdb.post_mortem(sys.exc_info()[2])
-            result = descr._backend_failure_recovery
+            result = cpu.assembler.guard_compat_recovery
         jitframe.jf_gcmap = lltype.nullptr(lltype.typeOf(jitframe.jf_gcmap).TO)
-        return result
+        p_arg.bchoices = bchoices
+        p_arg.jump_to = result
     return invoke_find_compatible
 
 def add_in_tree(bchoices, new_gcref, new_asmaddr):
@@ -175,7 +181,7 @@
     pairs_quicksort(addr, length)
     return bchoices
 
-def initial_bchoices(guard_compat_descr, initial_gcref):
+def initial_bchoices(guard_compat_descr):
     bchoices = lltype.malloc(BACKEND_CHOICES, 1)
     bchoices.bc_faildescr = cast_instance_to_gcref(guard_compat_descr)
     bchoices.bc_gc_table_tracer = lltype.nullptr(llmemory.GCREF.TO)   # (*)
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -981,10 +981,8 @@
         from rpython.jit.backend.llsupport import guard_compat
         # don't use _gcref_index here: we need our own index for
         # the _backend_choices object
-        c = op.getarg(1)
-        assert isinstance(c, ConstPtr)
         descr = op.getdescr()
-        bchoices = guard_compat.initial_bchoices(descr, c.value)
+        bchoices = guard_compat.initial_bchoices(descr)
         bcindex = len(self.gcrefs_output_list)
         gcref = lltype.cast_opaque_ptr(llmemory.GCREF, bchoices)
         self.gcrefs_output_list.append(gcref)
diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py
--- a/rpython/jit/backend/x86/guard_compat.py
+++ b/rpython/jit/backend/x86/guard_compat.py
@@ -126,8 +126,10 @@
 #     JMP *[RSP+array_element_2]         # may jump to guard_compat_recovery
 #
 #
-# invoke_find_compatible(bchoices, new_gcref, jitframe):
+# invoke_find_compatible(p_arg, jitframe):
 #     IN PSEUDO-CODE:
+#     bchoices = p_arg[0]
+#     new_gcref = p_arg[1]
 #     result = bchoices.bc_faildescr.find_compatible(cpu, new_gcref)
 #     if result == 0:
 #         result = descr._backend_failure_recovery
@@ -183,7 +185,6 @@
 
     mc = codebuf.MachineCodeBlockWrapper()
     mc.force_frame_size(frame_size)
-    mc.INT3()
     if IS_X86_32:    # save edi as an extra scratch register
         XXX
         mc.MOV_sr(3*WORD, rdi)
@@ -240,20 +241,21 @@
         mc.MOV_rs(rdi, 3*WORD)
 
     # The _backend_choices object is still referenced from [RSP+16]
-    # (which becomes [RSP+8] after the POP), where it is the first of a
-    # two-words array passed as argument to invoke_find_compatible().
-    # The second word is the value, from RAX, which we store now.
-    mc.MOV_sr(3*WORD, rax)                  # MOV [RSP+24], RAX
+    # (which becomes [RSP+8] after the POP), where it is the second of a
+    # three-words array passed as argument to invoke_find_compatible().
+    # The first word is the value, from RAX, which we store in (*)
+    # below.
 
     # restore RAX and RCX
     mc.MOV_rs(rcx, 1*WORD)                  # MOV RCX, [RSP+8]
+    mc.MOV_sr(1*WORD, rax)                  # MOV [RSP+8], RAX   (*)
     mc.POP_r(rax)                           # POP RAX
 
     # save all registers to the jitframe RBP
     assembler._push_all_regs_to_frame(mc, [], withfloats=True)
 
     if IS_X86_64:
-        mc.LEA_rs(rdi, 2 * WORD)            # LEA RDI, [RSP+8]
+        mc.MOV_rr(rdi, regloc.esp.value)    # MOV RDI, RSP
         mc.MOV_rr(regloc.esi.value,         # MOV RSI, RBP
                   regloc.ebp.value)
     elif IS_X86_32:
@@ -273,13 +275,13 @@
     # restore them all.
     assembler._pop_all_regs_from_frame(mc, [], withfloats=True)
 
-    # jump to 'array_element_2'.  In case this goes to
-    # guard_compat_recovery, we also reload the _backend_choices
-    # object from 'array_element_1' (the GC may have moved it, or
-    # it may be a completely new object).
+    # jump to the result, which is passed as the third word of the
+    # array.  In case this goes to guard_compat_recovery, we also reload
+    # the _backend_choices object from the second word of the array (the
+    # GC may have moved it, or it may be a completely new object).
     if IS_X86_64:
-        mc.MOV_rs(r11, 1*WORD)              # MOV R11, [RSP+8]
-        mc.JMP_s(2*WORD)                    # JMP *[RSP+16]
+        mc.MOV_rs(r11, 0)                   # MOV R11, [RSP]
+        mc.JMP_s(2 * WORD)                  # JMP *[RSP+16]
     elif IS_X86_32:
         XXX
         mc.JMP_s(0)
diff --git a/rpython/jit/backend/x86/test/test_compatible.py b/rpython/jit/backend/x86/test/test_compatible.py
--- a/rpython/jit/backend/x86/test/test_compatible.py
+++ b/rpython/jit/backend/x86/test/test_compatible.py
@@ -32,14 +32,16 @@
     mc.SUB(regloc.esp, regloc.imm(144 - 2*WORD)) # make a frame, and align stack
     mc.MOV(regloc.ebp, regloc.ecx)
     #
+    mc.MOV(regloc.ecx, regloc.imm(0xdddd))
     mc.PUSH(regloc.imm(0xaaaa))
     # jump to guard_compat_search_tree, but carefully: don't overwrite R11
     mc.MOV(regloc.esi, regloc.imm(cpu.assembler.guard_compat_search_tree))
     mc.JMP_r(regloc.esi.value)
+    mc.INT3()
     sequel = mc.get_relative_pos()
     #
     mc.force_frame_size(144)
-    mc.SUB(regloc.eax, regloc.edx)
+    mc.SUB(regloc.eax, regloc.ecx)
     mc.ADD(regloc.esp, regloc.imm(144 - 2*WORD))
     mc.POP(regloc.ebp)
     mc.RET()
@@ -61,8 +63,7 @@
         rawstart + 4 * WORD)
 
     guard_compat_descr = GuardCompatibleDescr()
-    bchoices = initial_bchoices(guard_compat_descr,
-                                rffi.cast(llmemory.GCREF, 111111))
+    bchoices = initial_bchoices(guard_compat_descr)
     llop.raw_store(lltype.Void, rawstart, 3 * WORD, bchoices)
 
     class FakeGuardToken:
@@ -78,6 +79,10 @@
                            lltype.nullptr(llmemory.GCREF.TO),
                            9999)
 
+    # fill in the first choice manually
+    bchoices.bc_list[0].gcref = r_uint(111111)
+    bchoices.bc_list[0].asmaddr = rawstart + sequel
+
     # ---- ready ----
 
     frame_info = lltype.malloc(jitframe.JITFRAMEINFO, flavor='raw')

From pypy.commits at gmail.com  Thu Feb  9 16:46:45 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Thu, 09 Feb 2017 13:46:45 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: Update my bio
Message-ID: <589ce345.125e1c0a.a60d0.2ed1@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r853:3ecc0193cbef
Date: 2017-02-09 16:46 -0500
http://bitbucket.org/pypy/pypy.org/changeset/3ecc0193cbef/

Log:	Update my bio

diff --git a/people.html b/people.html
--- a/people.html
+++ b/people.html
@@ -75,7 +75,7 @@
 <h1>Armin Rigo</h1>
 <img alt="image/people/arigo.png" src="image/people/arigo.png" />
 <p>Armin Rigo is a former researcher at the Heinrich-Heine Universitat
-Dusseldorf (Germany).  He studied Mathematics at the University
+D&#252;sseldorf (Germany).  He studied Mathematics at the University
 of Lausanne (Switzerland), obtained his Ph.D. in Logic and Set
 Theory at the Free University of Brussels (Belgium) in 2002, and
 worked at the University of Southampton (UK) until 2005.  He is
@@ -133,13 +133,13 @@
 <div class="section" id="alex-gaynor">
 <h1>Alex Gaynor</h1>
 <img alt="image/people/alex.jpg" src="image/people/alex.jpg" />
-<p>Alex is software engineer working for Rackspace, and living in San Francisco,
-CA. He's been a PyPy developer since 2010, and has worked on many parts of the
-codebase, including the JIT compiler's optimizers, the RPython translation
-toolchain, and the Python interpreter. In addition to his work on PyPy, Alex is
-also the creator of Topaz, a Ruby VM built on RPython, as well as a core
-developer of Django (a Python web framework) and CPython, as well as an
-member of the board of directors of the Python Software Foundation.</p>
+<p>Alex is software engineer living in Washington, DC. He's been a PyPy developer
+since 2010, and has worked on many parts of the codebase, including the JIT
+compiler's optimizers, the RPython translation toolchain, and the Python
+interpreter. In addition to his work on PyPy, Alex is also the creator of
+Topaz, a Ruby VM built on RPython and a core developer of Django (a Python web
+framework) and CPython, as well as a retired member of the board of directors
+of the Python Software Foundation.</p>
 </div>
 <div class="section" id="hakan-ardo">
 <h1>H&#229;kan Ard&#246;</h1>
@@ -152,9 +152,9 @@
 2003-2009 in the Mathematical Imaging Group. His thesis work consisted
 of designing image processing algorithms for traffic surveillance,
 aiming for a system that automatically measures the safety of an
-intersection or road segment. He is currently working part time as a
+intersection or road segment. He is currently working part-time as a
 postdoc at the Centre for Mathematical Sciences of Lund University
-continuing this work and part time as CTO with a spinoff company
+continuing this work and part-time as CTO with a spinoff company
 Cognimatics. His contributions to PyPy started 2010 and consists of
 the array module as well as work on the JIT compiler's trace optimizers.</p>
 </div>
diff --git a/source/people.txt b/source/people.txt
--- a/source/people.txt
+++ b/source/people.txt
@@ -9,7 +9,7 @@
 .. image:: image/people/arigo.png
 
 Armin Rigo is a former researcher at the Heinrich-Heine Universitat
-Dusseldorf (Germany).  He studied Mathematics at the University
+D&#252;sseldorf (Germany).  He studied Mathematics at the University
 of Lausanne (Switzerland), obtained his Ph.D. in Logic and Set
 Theory at the Free University of Brussels (Belgium) in 2002, and
 worked at the University of Southampton (UK) until 2005.  He is
@@ -80,13 +80,13 @@
 
 .. image:: image/people/alex.jpg
 
-Alex is software engineer working for Rackspace, and living in San Francisco,
-CA. He's been a PyPy developer since 2010, and has worked on many parts of the
-codebase, including the JIT compiler's optimizers, the RPython translation
-toolchain, and the Python interpreter. In addition to his work on PyPy, Alex is
-also the creator of Topaz, a Ruby VM built on RPython, as well as a core
-developer of Django (a Python web framework) and CPython, as well as an
-member of the board of directors of the Python Software Foundation.
+Alex is software engineer living in Washington, DC. He's been a PyPy developer
+since 2010, and has worked on many parts of the codebase, including the JIT
+compiler's optimizers, the RPython translation toolchain, and the Python
+interpreter. In addition to his work on PyPy, Alex is also the creator of
+Topaz, a Ruby VM built on RPython and a core developer of Django (a Python web
+framework) and CPython, as well as a retired member of the board of directors
+of the Python Software Foundation.
 
 H&#229;kan Ard&#246;
 ==========
@@ -101,9 +101,9 @@
 2003-2009 in the Mathematical Imaging Group. His thesis work consisted
 of designing image processing algorithms for traffic surveillance,
 aiming for a system that automatically measures the safety of an
-intersection or road segment. He is currently working part time as a
+intersection or road segment. He is currently working part-time as a
 postdoc at the Centre for Mathematical Sciences of Lund University
-continuing this work and part time as CTO with a spinoff company
+continuing this work and part-time as CTO with a spinoff company
 Cognimatics. His contributions to PyPy started 2010 and consists of
 the array module as well as work on the JIT compiler's trace optimizers.
 

From pypy.commits at gmail.com  Fri Feb 10 05:00:54 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 10 Feb 2017 02:00:54 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: rename variable because term.h
 defines the same macro (insane to call a define "lines" if you ask me),
 change key while walking gcroots
Message-ID: <589d8f56.eb86df0a.61e36.9953@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90030:519dbbf54f3d
Date: 2017-02-10 11:00 +0100
http://bitbucket.org/pypy/pypy/changeset/519dbbf54f3d/

Log:	rename variable because term.h defines the same macro (insane to
	call a define "lines" if you ask me), change key while walking
	gcroots

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,6 +1,5 @@
 import sys, os
-from rpython.rlib.objectmodel import (specialize, we_are_translated,
-        not_rpython, we_are_translated_to_c)
+from rpython.rlib.objectmodel import specialize, we_are_translated, not_rpython
 from rpython.rlib import jit, rposix, rgc
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -10,6 +10,8 @@
 #  include "rvmprof.h"
 #endif
 
+
+
 #include "shared/vmprof_get_custom_offset.h"
 #ifdef VMPROF_UNIX
 #include "shared/vmprof_main.h"
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.h b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.h
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.h
@@ -10,7 +10,7 @@
 int vmp_ignore_ip(intptr_t ip);
 int vmp_binary_search_ranges(intptr_t ip, intptr_t * l, int count);
 int vmp_native_symbols_read(void);
-void vmp_profile_lines(int lines);
+void vmp_profile_lines(int);
 int vmp_profiles_python_lines(void);
 
 int vmp_ignore_symbol_count(void);
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -17,7 +17,7 @@
 static long prepare_interval_usec = 0;
 static long profile_interval_usec = 0;
 
-static int opened_profile(const char *interp_name, int memory, int lines, int native);
+static int opened_profile(const char *interp_name, int memory, int proflines, int native);
 
 #ifdef VMPROF_UNIX
 static struct profbuf_s *volatile current_codes;
@@ -58,7 +58,7 @@
 
 RPY_EXTERN
 char *vmprof_init(int fd, double interval, int memory,
-                  int lines, const char *interp_name, int native)
+                  int proflines, const char *interp_name, int native)
 {
     if (!(interval >= 1e-6 && interval < 1.0)) {   /* also if it is NaN */
         return "bad value for 'interval'";
@@ -79,14 +79,14 @@
 #endif
     assert(fd >= 0);
     vmp_set_profile_fileno(fd);
-    if (opened_profile(interp_name, memory, lines, native) < 0) {
+    if (opened_profile(interp_name, memory, proflines, native) < 0) {
         vmp_set_profile_fileno(0);
         return strerror(errno);
     }
     return NULL;
 }
 
-static int opened_profile(const char *interp_name, int memory, int lines, int native)
+static int opened_profile(const char *interp_name, int memory, int proflines, int native)
 {
     int success;
     int bits;
@@ -105,7 +105,7 @@
     header.interp_name[0] = MARKER_HEADER;
     header.interp_name[1] = '\x00';
     header.interp_name[2] = VERSION_TIMESTAMP;
-    header.interp_name[3] = memory*PROFILE_MEMORY + lines*PROFILE_LINES + \
+    header.interp_name[3] = memory*PROFILE_MEMORY + proflines*PROFILE_LINES + \
                             native*PROFILE_NATIVE;
 #ifdef RPYTHON_VMPROF
     header.interp_name[3] += PROFILE_RPYTHON;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -51,7 +51,7 @@
 /************************************************************/
 
 static void *(*mainloop_get_virtual_ip)(char *) = 0;
-static int opened_profile(const char *interp_name, int memory, int lines, int native);
+static int opened_profile(const char *interp_name, int memory, int proflines, int native);
 static void flush_codes(void);
 
 /************************************************************/
diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py
--- a/rpython/translator/c/gcc/trackgcroot.py
+++ b/rpython/translator/c/gcc/trackgcroot.py
@@ -368,8 +368,8 @@
         def schedule(insn, state):
             for previnsn in insn.previous_insns:
                 key = previnsn, state
-                if key not in seen:
-                    seen.add(key)
+                if previnsn not in seen:
+                    seen.add(previnsn)
                     pending.append(key)
         schedule(initial_insn, initial_state)
         while pending:

From pypy.commits at gmail.com  Fri Feb 10 05:18:47 2017
From: pypy.commits at gmail.com (Raemi)
Date: Fri, 10 Feb 2017 02:18:47 -0800 (PST)
Subject: [pypy-commit] stmgc default: be smarter about finding a source page
 in the signal handler
Message-ID: <589d9387.0e821c0a.55f2.4c28@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2012:6cd15231609a
Date: 2017-02-10 11:18 +0100
http://bitbucket.org/pypy/stmgc/changeset/6cd15231609a/

Log:	be smarter about finding a source page in the signal handler

	It's unclear if this approach improves performance, but the previous
	strategy of simply finding the most recent revision sounded too
	simple. If only for the additional comment, the new approach makes
	more sense intuitively.

diff --git a/c8/stm/signal_handler.c b/c8/stm/signal_handler.c
--- a/c8/stm/signal_handler.c
+++ b/c8/stm/signal_handler.c
@@ -5,6 +5,7 @@
 
 
 
+
 static void setup_signal_handler(void)
 {
     struct sigaction act;
@@ -76,19 +77,37 @@
 
     assert(get_page_status_in(my_segnum, pagenum) == PAGE_NO_ACCESS);
 
-    /* find who has the most recent revision of our page */
+    /* find a suitable page to copy from in other segments:
+     * suitable means:
+     *  - if there is a revision around >= target_rev, get the oldest >= target_rev.
+     *  - otherwise find most recent revision
+     * Note: simply finding the most recent revision would be a conservative strategy, but
+     *       requires going back in time more often (see below)
+     */
     int copy_from_segnum = -1;
-    uint64_t most_recent_rev = 0;
+    uint64_t copy_from_rev = 0;
+    uint64_t target_rev = STM_PSEGMENT->last_commit_log_entry->rev_num;
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (i == my_segnum)
             continue;
 
         struct stm_commit_log_entry_s *log_entry;
         log_entry = get_priv_segment(i)->last_commit_log_entry;
-        if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS
-            && (copy_from_segnum == -1 || log_entry->rev_num > most_recent_rev)) {
+
+        /* - if not found anything, initialise copy_from_rev
+         * - else if target_rev is higher than everything we found, find newest among them
+         * - else: find revision that is as close to target_rev as possible         */
+        bool accessible = get_page_status_in(i, pagenum) != PAGE_NO_ACCESS;
+        bool uninit = copy_from_segnum == -1;
+        bool find_most_recent = copy_from_rev < target_rev && log_entry->rev_num > copy_from_rev;
+        bool find_closest = copy_from_rev >= target_rev && (
+            log_entry->rev_num - target_rev < copy_from_rev - target_rev);
+
+        if (accessible && (uninit || find_most_recent || find_closest)) {
             copy_from_segnum = i;
-            most_recent_rev = log_entry->rev_num;
+            copy_from_rev = log_entry->rev_num;
+            if (copy_from_rev == target_rev)
+                break;
         }
     }
     OPT_ASSERT(copy_from_segnum != my_segnum);
@@ -118,21 +137,20 @@
     /* if there were modifications in the page, revert them. */
     copy_bk_objs_in_page_from(copy_from_segnum, pagenum, false);
 
-    /* we need to go from 'src_version' to 'target_version'.  This
+    dprintf(("handle_segfault_in_page: rev %lu to rev %lu\n",
+             copy_from_rev, target_rev));
+    /* we need to go from 'copy_from_rev' to 'target_rev'.  This
        might need a walk into the past. */
-    struct stm_commit_log_entry_s *src_version, *target_version;
-    src_version = get_priv_segment(copy_from_segnum)->last_commit_log_entry;
-    target_version = STM_PSEGMENT->last_commit_log_entry;
+    if (copy_from_rev > target_rev) {
+        /* adapt revision of page to our revision:
+           if our rev is higher than the page we copy from, everything
+           is fine as we never read/modified the page anyway */
+        struct stm_commit_log_entry_s *src_version, *target_version;
+        src_version = get_priv_segment(copy_from_segnum)->last_commit_log_entry;
+        target_version = STM_PSEGMENT->last_commit_log_entry;
 
-
-    dprintf(("handle_segfault_in_page: rev %lu to rev %lu\n",
-             src_version->rev_num, target_version->rev_num));
-    /* adapt revision of page to our revision:
-       if our rev is higher than the page we copy from, everything
-       is fine as we never read/modified the page anyway
-     */
-    if (src_version->rev_num > target_version->rev_num)
         go_to_the_past(pagenum, src_version, target_version);
+    }
 
     release_modification_lock_set(to_lock, my_segnum);
     release_all_privatization_locks();

From pypy.commits at gmail.com  Fri Feb 10 10:38:28 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 10 Feb 2017 07:38:28 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: call dyn_register_jit_page when a
 loop or bridge is compiled
Message-ID: <589dde74.90b1df0a.4f04b.0e6b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90032:1b6119b16048
Date: 2017-02-10 16:37 +0100
http://bitbucket.org/pypy/pypy/changeset/1b6119b16048/

Log:	call dyn_register_jit_page when a loop or bridge is compiled

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -501,11 +501,17 @@
                                       'compiling', None, name, memo)
     _log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE_OPT, metainterp_sd, None)
     _log.write(inputargs, operations)
-    return metainterp_sd.cpu.compile_loop(inputargs,
+    asminfo = metainterp_sd.cpu.compile_loop(inputargs,
                                           operations, looptoken,
                                           jd_id=jd_id, unique_id=unique_id,
-                                          log=log, name=name,
-                                          logger=metainterp_sd.jitlog)
+                                          log=log, name=name, logger=metainterp_sd.jitlog)
+
+    vmprof = metainterp_sd.vmprof
+    if vmprof:
+        vmprof.dyn_register_jit_page(asminfo.asmaddr,
+                                     asminfo.asmaddr+asminfo.asmlen, 1)
+
+    return asminfo
 
 def do_compile_bridge(metainterp_sd, faildescr, inputargs, operations,
                       original_loop_token, log=True, memo=None):
@@ -515,9 +521,14 @@
     _log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE_OPT, metainterp_sd, None)
     _log.write(inputargs, operations)
     assert isinstance(faildescr, AbstractFailDescr)
-    return metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations,
+    asminfo = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations,
                                             original_loop_token, log=log,
                                             logger=metainterp_sd.jitlog)
+    vmprof = metainterp_sd.vmprof
+    if vmprof:
+        vmprof.dyn_register_jit_page(asminfo.asmaddr,
+                                     asminfo.asmaddr+asminfo.asmlen, 0)
+    return asminfo
 
 def forget_optimization_info(lst, reset_values=False):
     for item in lst:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1823,6 +1823,12 @@
         self.jitlog.logger_noopt = self.logger_noopt
         self.jitlog.logger_ops = self.logger_ops
 
+        from rpython.rlib.rvmprof import rvmprof, cintf
+        try:
+            self.vmprof = rvmprof._get_vmprof()
+        except cintf.VMProfPlatformUnsupported:
+            self.vmprof = None
+
         self.profiler = ProfilerClass()
         self.profiler.cpu = cpu
         self.warmrunnerdesc = warmrunnerdesc
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -160,6 +160,17 @@
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
 
+
+    def dyn_register_jit_page(self, addr, end_addr, loop, name=None):
+        if name is None:
+            cname = rffi.cast(rffi.CHARP, 0)
+        else:
+            cname = rffi.str2charp(name)
+        return self.cintf.vmp_dyn_register_jit_page(addr, end_addr, cname)
+
+    def dyn_cancel(self, ref):
+        self.cintf.vmp_dyn_cancel(ref)
+
     def _write_code_registration(self, uid, name):
         assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
             "the name must be 'class:func_name:func_line:filename' "
@@ -250,9 +261,6 @@
 def _was_registered(CodeClass):
     return hasattr(CodeClass, '_vmprof_unique_id')
 
-def register_jit_page(addr, end_addr, splits):
-    pass
-
 _vmprof_instance = None
 
 @specialize.memo()

From pypy.commits at gmail.com  Fri Feb 10 10:38:26 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 10 Feb 2017 07:38:26 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: add new test that registers
 dynamically allocated code for libunwind
Message-ID: <589dde72.cca81c0a.517d5.2415@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90031:8ed69cf08956
Date: 2017-02-10 16:01 +0100
http://bitbucket.org/pypy/pypy/changeset/8ed69cf08956/

Log:	add new test that registers dynamically allocated code for libunwind

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -250,6 +250,8 @@
 def _was_registered(CodeClass):
     return hasattr(CodeClass, '_vmprof_unique_id')
 
+def register_jit_page(addr, end_addr, splits):
+    pass
 
 _vmprof_instance = None
 
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
@@ -0,0 +1,120 @@
+#include "vmp_dynamic.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+static int g_dyn_entry_count = 0;
+static int g_has_holes = -1;
+static int g_dyn_entry_count_max = 0;
+static unw_dyn_info_t ** g_dyn_entries = 0;
+
+int vmp_dyn_teardown(void)
+{
+    if (g_dyn_entries != NULL) {
+        free(g_dyn_entries);
+    }
+    g_dyn_entry_count = 0;
+    g_dyn_entry_count_max = 0;
+    g_has_holes = -1;
+    return 0;
+}
+
+static void _vmp_dyn_resize(void) {
+    if (g_dyn_entry_count_max == 0) {
+        g_dyn_entry_count_max = 128;
+        g_dyn_entries = (unw_dyn_info_t**)calloc(sizeof(unw_dyn_info_t*), 128);
+    }
+
+    if (g_dyn_entry_count + 1 >= g_dyn_entry_count_max) {
+        g_dyn_entry_count_max *= 2;
+        g_dyn_entries = (unw_dyn_info_t**)realloc(g_dyn_entries, sizeof(unw_dyn_info_t*) * g_dyn_entry_count_max);
+        memset(g_dyn_entries + g_dyn_entry_count, 0,
+               sizeof(unw_dyn_info_t*)*g_dyn_entry_count_max - g_dyn_entry_count);
+    }
+}
+
+static unw_dyn_info_t * _vmp_alloc_dyn_info(int * reference)
+{
+    unw_dyn_info_t * u;
+
+    u = (unw_dyn_info_t*)malloc(sizeof(unw_dyn_info_t));
+
+    int i = 0;
+    int ref = -1;
+    if (g_has_holes >= 0) {
+        i = g_has_holes;
+        while (i < g_dyn_entry_count) {
+            if (g_dyn_entries[i] == NULL) {
+                ref = i;
+                g_has_holes += 1;
+            }
+        }
+        if (i == g_dyn_entry_count) {
+            _vmp_dyn_resize();
+            ref = g_dyn_entry_count;
+            g_dyn_entry_count++;
+        }
+    } else {
+        _vmp_dyn_resize();
+        ref = g_dyn_entry_count;
+        g_dyn_entry_count++;
+    }
+    assert(ref != -1 && "ref position MUST be found");
+    g_dyn_entries[ref] = u;
+    *reference = ref;
+
+    return u;
+}
+
+static void _vmp_free_dyn_info(unw_dyn_info_t * u)
+{
+    free(u);
+}
+
+int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr,
+                              const char * name)
+{
+    char * name_cpy = NULL;
+    int ref = -1;
+    unw_dyn_info_t * u = _vmp_alloc_dyn_info(&ref);
+    if (ref == -1) {
+        return -1; // fail, could not alloc
+    }
+    u->start_ip = (unw_word_t)addr;
+    u->end_ip = (unw_word_t)end_addr;
+    u->format = UNW_INFO_FORMAT_DYNAMIC;
+    if (name != NULL) {
+        name_cpy = strdup(name);
+    }
+    unw_dyn_proc_info_t * ip = (unw_dyn_proc_info_t*)&(u->u);
+    ip->name_ptr = (unw_word_t)name_cpy;
+    ip->handler = 0;
+    ip->flags = 0;
+    ip->regions = NULL;
+
+    _U_dyn_register(u);
+
+    return ref;
+}
+
+int vmp_dyn_cancel(int ref) {
+    unw_dyn_info_t * u;
+
+    if (ref >= g_dyn_entry_count || ref < 0) {
+        return 1;
+    }
+    
+    u = g_dyn_entries[ref];
+    if (u != NULL) {
+        g_dyn_entries[ref] = NULL;
+        if (g_has_holes > ref) {
+            g_has_holes = ref;
+        }
+
+        _U_dyn_cancel(u);
+    }
+
+    _vmp_free_dyn_info(u);
+    return 0;
+}
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <stdint.h>
+#include <libunwind.h>
+
+int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr,
+                              const char * name);
+int vmp_dyn_cancel(int ref);
+int vmp_dyn_teardown(void);
+
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -241,7 +241,7 @@
             // this is possible because compiler align to 8 bytes.
             //
 #ifdef PYPY_JIT_CODEMAP
-            if (func_addr == 0 && top_most_frame->kind == VMPROF_JITTED_TAG) {
+            if (top_most_frame->kind == VMPROF_JITTED_TAG) {
                 intptr_t pc = ((intptr_t*)(top_most_frame->value - sizeof(intptr_t)))[0];
                 depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
                 frame = FRAME_STEP(frame);
@@ -291,9 +291,7 @@
     // the native stack?
 #ifdef RPYTHON_VMPROF
     if (strstr(name, "libpypy-c.so") != NULL
-        || strstr(name, "pypy-c") != NULL
-        || strstr(name, "pypy") != NULL) {
-        printf("ignoring %s\n", name);
+        || strstr(name, "pypy-c") != NULL) {
         return 1;
     }
 #else
diff --git a/rpython/rlib/rvmprof/test/test_dynamic.py b/rpython/rlib/rvmprof/test/test_dynamic.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/test/test_dynamic.py
@@ -0,0 +1,47 @@
+import py
+import sys
+try:
+    import cffi
+except ImportError:
+    py.test.skip('cffi required')
+
+from rpython.rlib import rvmprof
+srcdir = py.path.local(rvmprof.__file__).join("..", "src")
+
+
+ at py.test.mark.skipif("sys.platform == 'win32'")
+class TestDirect(object):
+    def setup_class(clz):
+        ffi = cffi.FFI()
+        ffi.cdef("""
+        int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr, const char * name);
+        int vmp_dyn_cancel(int ref);
+        """)
+
+        with open(str(srcdir.join("shared/vmp_dynamic.c"))) as fd:
+            ffi.set_source("rpython.rlib.rvmprof.test._test_dynamic", fd.read(),
+                    include_dirs=[str(srcdir.join('shared'))],
+                    libraries=['unwind'])
+
+        ffi.compile(verbose=True)
+
+        from rpython.rlib.rvmprof.test import _test_dynamic
+        clz.lib = _test_dynamic.lib
+        clz.ffi = _test_dynamic.ffi
+
+    def test_register_dynamic_code(self):
+        lib = self.lib
+        ffi = self.ffi
+
+        assert 1 == lib.vmp_dyn_cancel(100)
+        assert 1 == lib.vmp_dyn_cancel(0)
+        assert 1 == lib.vmp_dyn_cancel(-1)
+
+        s = ffi.new("char[]", "hello jit compiler")
+        assert 0 == lib.vmp_dyn_register_jit_page(0x100, 0x200, ffi.NULL)
+        assert 1 == lib.vmp_dyn_register_jit_page(0x200, 0x300, s)
+
+        lib.vmp_dyn_cancel(0)
+        lib.vmp_dyn_cancel(1)
+
+
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -237,3 +237,4 @@
     finally:
         assert os.path.exists(tmpfilename)
         os.unlink(tmpfilename)
+

From pypy.commits at gmail.com  Fri Feb 10 10:51:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 07:51:31 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: Pass the first test
Message-ID: <589de183.4a6b1c0a.8f862.c260@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90033:2e67cb4a66b5
Date: 2017-02-10 16:49 +0100
http://bitbucket.org/pypy/pypy/changeset/2e67cb4a66b5/

Log:	Pass the first test

diff --git a/rpython/jit/backend/llsupport/guard_compat.py b/rpython/jit/backend/llsupport/guard_compat.py
--- a/rpython/jit/backend/llsupport/guard_compat.py
+++ b/rpython/jit/backend/llsupport/guard_compat.py
@@ -103,7 +103,7 @@
 
 INVOKE_FIND_COMPATIBLE_FUNC = lltype.Ptr(lltype.FuncType(
                 [lltype.Ptr(P_ARG), lltype.Ptr(jitframe.JITFRAME)],
-                lltype.Signed))
+                lltype.Void))
 
 @specialize.memo()
 def make_invoke_find_compatible(cpu):
diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py
--- a/rpython/jit/backend/x86/guard_compat.py
+++ b/rpython/jit/backend/x86/guard_compat.py
@@ -280,7 +280,7 @@
     # the _backend_choices object from the second word of the array (the
     # GC may have moved it, or it may be a completely new object).
     if IS_X86_64:
-        mc.MOV_rs(r11, 0)                   # MOV R11, [RSP]
+        mc.MOV_rs(r11, 1 * WORD)            # MOV R11, [RSP]
         mc.JMP_s(2 * WORD)                  # JMP *[RSP+16]
     elif IS_X86_32:
         XXX
diff --git a/rpython/jit/backend/x86/test/test_compatible.py b/rpython/jit/backend/x86/test/test_compatible.py
--- a/rpython/jit/backend/x86/test/test_compatible.py
+++ b/rpython/jit/backend/x86/test/test_compatible.py
@@ -1,5 +1,7 @@
 import random
 from rpython.jit.backend.x86.guard_compat import *
+from rpython.jit.backend.x86.arch import FRAME_FIXED_SIZE, PASS_ON_MY_FRAME
+from rpython.jit.backend.x86.arch import DEFAULT_FRAME_BYTES
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 from rpython.jit.backend.detect_cpu import getcpuclass
 from rpython.jit.metainterp.test import test_compatible
@@ -18,8 +20,14 @@
     for i in range(4 * WORD):
         mc.writechar('\x00')   # 4 gctable entries; 'bchoices' will be #3
     #
+    # extracted from _call_header()
+    mc.SUB_ri(regloc.esp.value, FRAME_FIXED_SIZE * WORD)
+    mc.MOV_sr(PASS_ON_MY_FRAME * WORD, regloc.ebp.value)
+    for i, loc in enumerate(cpu.CALLEE_SAVE_REGISTERS):
+        mc.MOV_sr((PASS_ON_MY_FRAME + i + 1) * WORD, loc.value)
+    #
     if IS_X86_64:
-        mc.MOV(regloc.ecx, regloc.edx)    # jitframe
+        mc.MOV(regloc.ebp, regloc.edx)    # jitframe
         mc.MOV(regloc.r11, regloc.edi)    # _backend_choices
         mc.MOV(regloc.eax, regloc.esi)    # guarded value
     elif IS_X86_32:
@@ -28,10 +36,6 @@
         mc.MOV_rs(regloc.eax.value, 8)
         mc.MOV_rs(regloc.ecx.value, 12)
     #
-    mc.PUSH(regloc.ebp)
-    mc.SUB(regloc.esp, regloc.imm(144 - 2*WORD)) # make a frame, and align stack
-    mc.MOV(regloc.ebp, regloc.ecx)
-    #
     mc.MOV(regloc.ecx, regloc.imm(0xdddd))
     mc.PUSH(regloc.imm(0xaaaa))
     # jump to guard_compat_search_tree, but carefully: don't overwrite R11
@@ -40,21 +44,23 @@
     mc.INT3()
     sequel = mc.get_relative_pos()
     #
-    mc.force_frame_size(144)
+    mc.force_frame_size(DEFAULT_FRAME_BYTES)
     mc.SUB(regloc.eax, regloc.ecx)
-    mc.ADD(regloc.esp, regloc.imm(144 - 2*WORD))
-    mc.POP(regloc.ebp)
-    mc.RET()
+    #
+    def call_footer():
+        mc.MOV_rs(regloc.ebp.value, PASS_ON_MY_FRAME * WORD)
+        for i, loc in enumerate(cpu.CALLEE_SAVE_REGISTERS):
+            mc.MOV_rs(loc.value, (PASS_ON_MY_FRAME + i + 1) * WORD)
+        mc.ADD_ri(regloc.esp.value, FRAME_FIXED_SIZE * WORD)
+        mc.RET()
+    call_footer()
     #
     extra_paths = []
     for i in range(11):
-        mc.force_frame_size(144)
+        mc.force_frame_size(DEFAULT_FRAME_BYTES)
         extra_paths.append(mc.get_relative_pos())
         mc.MOV(regloc.eax, regloc.imm(1000000 + i))
-        mc.ADD(regloc.esp, regloc.imm(144 - 2*WORD))
-        mc.POP(regloc.ebp)
-        mc.RET()
-    failure = extra_paths[10]
+        call_footer()
     rawstart = mc.materialize(cpu, [])
     print 'rawstart:', hex(rawstart)
     call_me = rffi.cast(lltype.Ptr(lltype.FuncType(
@@ -110,10 +116,11 @@
         gcref = rffi.cast(llmemory.GCREF, 123456 + i)
         print 'calling with a gcref never seen before'
         res = call_me(bchoices, gcref, frame)
-        assert res == 1000010
+        assert res == rffi.cast(lltype.Signed, frame)
         assert len(seen) == 1 + i
         assert bchoices.bc_most_recent.gcref == 123456 + i
-        assert bchoices.bc_most_recent.asmaddr == rawstart + failure
+        assert bchoices.bc_most_recent.asmaddr == (
+            cpu.assembler.guard_compat_recovery)
 
     # ---- grow bchoices ----
 

From pypy.commits at gmail.com  Fri Feb 10 10:51:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 07:51:32 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: pass the second test
Message-ID: <589de184.8688df0a.c2ac2.1237@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90034:d16698c1a52b
Date: 2017-02-10 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/d16698c1a52b/

Log:	pass the second test

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1978,7 +1978,7 @@
         if guardtok.guard_compatible():
             assert startpos == self.mc.get_relative_pos()
             guard_compat.generate_recovery_stub(self, guardtok)
-            xXXXx
+            return startpos
         #
         self.push_from_gc_table(faildescrindex)
         self.push_gcmap(self.mc, guardtok.gcmap, push=True)
diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py
--- a/rpython/jit/backend/x86/guard_compat.py
+++ b/rpython/jit/backend/x86/guard_compat.py
@@ -304,6 +304,7 @@
 
 
 def generate_recovery_stub(assembler, guard_token):
+    rax = regloc.eax.value
     r11 = regloc.r11.value
     frame_size = DEFAULT_FRAME_BYTES
 

From pypy.commits at gmail.com  Fri Feb 10 11:32:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 08:32:14 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: Fixes. Now
 x86/test/test_compatible crashes much later
Message-ID: <589deb0e.2da9df0a.6b86b.222d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90036:335bfa51255f
Date: 2017-02-10 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/335bfa51255f/

Log:	Fixes. Now x86/test/test_compatible crashes much later

diff --git a/rpython/jit/backend/llsupport/guard_compat.py b/rpython/jit/backend/llsupport/guard_compat.py
--- a/rpython/jit/backend/llsupport/guard_compat.py
+++ b/rpython/jit/backend/llsupport/guard_compat.py
@@ -116,7 +116,7 @@
                 result = cpu.assembler.guard_compat_recovery
             else:
                 if result == -1:
-                    result = descr.adr_jump_offset
+                    result = cpu.assembler.guard_compat_continue_running(descr)
                 bchoices = add_in_tree(bchoices, new_gcref, result)
                 # ---no GC operation---
                 choices_addr = descr._backend_choices_addr  # GC table
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2084,6 +2084,12 @@
     def _build_guard_compat_slowpath(self):
         guard_compat.build_once(self)
 
+    def guard_compat_continue_running(self, descr):
+        # Return the code address that guard_compat.find_compatible() must
+        # use for the case "continue executing the same piece of code".
+        # It is the address just after the GUARD_COMPATIBLE instruction.
+        return descr.adr_jump_offset + 4
+
     def genop_finish(self, op, arglocs, result_loc):
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         if len(arglocs) > 0:
diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py
--- a/rpython/jit/backend/x86/guard_compat.py
+++ b/rpython/jit/backend/x86/guard_compat.py
@@ -78,7 +78,7 @@
 # assembler.py.
 #
 #   guard_compat_recovery:
-#     PUSH R11
+#     PUSH [R11 + bc_faildescr]
 #     PUSH [R11 + bc_gcmap]
 #     JMP target
 #
@@ -294,9 +294,10 @@
     r11 = regloc.r11.value
     mc = codebuf.MachineCodeBlockWrapper()
 
-    ofs = _real_number(BCGCMAP)
-    mc.PUSH_r(r11)
-    mc.PUSH_m((r11, ofs))
+    ofs1 = _real_number(BCFAILDESCR)
+    ofs2 = _real_number(BCGCMAP)
+    mc.PUSH_m((r11, ofs1))
+    mc.PUSH_m((r11, ofs2))
     target = assembler.get_target_for_failure_recovery_of_guard_compat()
     mc.JMP(regloc.imm(target))
 
@@ -329,30 +330,3 @@
 
     ofs = _real_number(BCSEARCHTREE)
     mc.JMP_m((r11, ofs))                    # JMP *[R11 + bc_search_tree]
-
-
-#def generate_guard_compatible(assembler, guard_token, reg, reg2, gctable_index):
-#    mc = assembler.mc
-#    mc.CMP_rr(reg, reg2)                    # CMP reg, reg2
-#    mc.J_il8(rx86.Conditions['E'], 0)       # JE sequel
-#    je_location = mc.get_relative_pos()
-#
-#    self.push_from_gc_table(guard_token.faildescrindex)
-#    mc.JMP(regloc.imm(assembler.guard_compat_second_case))
-#
-#    padding_end = start_pos + size_general_case - 2
-#    while mc.get_relative_pos() < padding_end:
-#        mc.INT3()
-#
-#    padding_end = mc.get_relative_pos()    # in case it is actually bigger
-#    block_size = padding_end - start_pos + 2
-#    assert 0 < block_size <= 255
-#    mc.writechar(chr(block_size))
-#    assert 0 <= reg <= 15 and 0 <= reg2 <= 15
-#    mc.writechar(chr((reg2 << 4) | reg))
-#
-#    # abuse this field to store the 'sequel' relative offset
-#    guard_token.pos_jump_offset = mc.get_relative_pos()
-#    guard_token.guard_compat_bindex = gctable_index
-#    guard_token..............
-#    assembler.pending_guard_tokens.append(guard_token)

From pypy.commits at gmail.com  Fri Feb 10 11:32:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 08:32:12 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: in-progress
Message-ID: <589deb0c.b086df0a.ceffa.1df6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90035:291ea6b8def5
Date: 2017-02-10 17:19 +0100
http://bitbucket.org/pypy/pypy/changeset/291ea6b8def5/

Log:	in-progress

diff --git a/rpython/jit/backend/llsupport/guard_compat.py b/rpython/jit/backend/llsupport/guard_compat.py
--- a/rpython/jit/backend/llsupport/guard_compat.py
+++ b/rpython/jit/backend/llsupport/guard_compat.py
@@ -97,19 +97,16 @@
     return rffi.cast(lltype.Unsigned, gcref)
 
 
-P_ARG = lltype.Struct('P_ARG', ('new_gcref', llmemory.GCREF),
-                               ('bchoices', lltype.Ptr(BACKEND_CHOICES)),
-                               ('jump_to', lltype.Signed))
-
 INVOKE_FIND_COMPATIBLE_FUNC = lltype.Ptr(lltype.FuncType(
-                [lltype.Ptr(P_ARG), lltype.Ptr(jitframe.JITFRAME)],
+                [rffi.SIGNEDP, lltype.Ptr(jitframe.JITFRAME)],
                 lltype.Void))
 
 @specialize.memo()
 def make_invoke_find_compatible(cpu):
     def invoke_find_compatible(p_arg, jitframe):
-        bchoices = p_arg.bchoices
-        new_gcref = p_arg.new_gcref
+        new_gcref = rffi.cast(llmemory.GCREF, p_arg[0])
+        bchoices = rffi.cast(lltype.Ptr(BACKEND_CHOICES), p_arg[1])
+        # ---GC operations cannot occur above---
         descr = bchoices.bc_faildescr
         descr = cast_gcref_to_instance(GuardCompatibleDescr, descr)
         try:
@@ -136,8 +133,9 @@
                 pdb.post_mortem(sys.exc_info()[2])
             result = cpu.assembler.guard_compat_recovery
         jitframe.jf_gcmap = lltype.nullptr(lltype.typeOf(jitframe.jf_gcmap).TO)
-        p_arg.bchoices = bchoices
-        p_arg.jump_to = result
+        # ---GC operations cannot occur below---
+        p_arg[0] = result
+        p_arg[1] = rffi.cast(lltype.Signed, bchoices)
     return invoke_find_compatible
 
 def add_in_tree(bchoices, new_gcref, new_asmaddr):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -785,7 +785,6 @@
                 guard_compat.patch_guard_compatible(
                     tok, self._addr_from_gc_table,
                     self.gc_table_tracer, self.guard_compat_search_tree)
-                continue
             descr = tok.faildescr
             if descr.loop_version():
                 continue # patch them later
diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py
--- a/rpython/jit/backend/x86/guard_compat.py
+++ b/rpython/jit/backend/x86/guard_compat.py
@@ -242,7 +242,7 @@
 
     # The _backend_choices object is still referenced from [RSP+16]
     # (which becomes [RSP+8] after the POP), where it is the second of a
-    # three-words array passed as argument to invoke_find_compatible().
+    # two-words array passed as argument to invoke_find_compatible().
     # The first word is the value, from RAX, which we store in (*)
     # below.
 
@@ -275,13 +275,13 @@
     # restore them all.
     assembler._pop_all_regs_from_frame(mc, [], withfloats=True)
 
-    # jump to the result, which is passed as the third word of the
+    # jump to the result, which is returned as the first word of the
     # array.  In case this goes to guard_compat_recovery, we also reload
     # the _backend_choices object from the second word of the array (the
     # GC may have moved it, or it may be a completely new object).
     if IS_X86_64:
-        mc.MOV_rs(r11, 1 * WORD)            # MOV R11, [RSP]
-        mc.JMP_s(2 * WORD)                  # JMP *[RSP+16]
+        mc.MOV_rs(r11, 1 * WORD)            # MOV R11, [RSP+8]
+        mc.JMP_s(0)                         # JMP *[RSP]
     elif IS_X86_32:
         XXX
         mc.JMP_s(0)

From pypy.commits at gmail.com  Fri Feb 10 11:34:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 08:34:44 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: fix, test_compatible passes
Message-ID: <589deba4.d30f1c0a.ef651.cbae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90037:c322bff3811f
Date: 2017-02-10 17:34 +0100
http://bitbucket.org/pypy/pypy/changeset/c322bff3811f/

Log:	fix, test_compatible passes

diff --git a/rpython/jit/backend/llsupport/guard_compat.py b/rpython/jit/backend/llsupport/guard_compat.py
--- a/rpython/jit/backend/llsupport/guard_compat.py
+++ b/rpython/jit/backend/llsupport/guard_compat.py
@@ -151,7 +151,9 @@
         new_bchoices = lltype.malloc(BACKEND_CHOICES, length * 2 + 1)
         # --- no GC below: it would mess up the order of bc_list ---
         new_bchoices.bc_faildescr = bchoices.bc_faildescr
+        new_bchoices.bc_gcmap = bchoices.bc_gcmap
         new_bchoices.bc_gc_table_tracer = bchoices.bc_gc_table_tracer
+        new_bchoices.bc_search_tree = bchoices.bc_search_tree
         new_bchoices.bc_most_recent.gcref = bchoices.bc_most_recent.gcref
         new_bchoices.bc_most_recent.asmaddr = bchoices.bc_most_recent.asmaddr
         i = 0

From pypy.commits at gmail.com  Fri Feb 10 11:43:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 08:43:51 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: fix test_rewrite
Message-ID: <589dedc7.d30f1c0a.ef651.ce23@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90038:73fc262b3645
Date: 2017-02-10 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/73fc262b3645/

Log:	fix test_rewrite

diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1439,19 +1439,28 @@
 
     def test_guard_compatible(self):
         from rpython.jit.backend.llsupport import guard_compat
-        self.check_rewrite("""
-            [p0]
-            guard_compatible(p0, ConstPtr(myR1)) []
-            guard_compatible(p0, ConstPtr(myR1)) []
-            jump()
-        """, """
-            [p0]
-            guard_compatible(p0, 0) []
-            guard_compatible(p0, 2) []    # no sharing the number
-            jump()
-        """)
-        assert len(self.gcrefs) == 4
-        for i in [0, 2]:
-            # type-checking
+        org_GuardCompatibleDescr = guard_compat.GuardCompatibleDescr
+        try:
+            guard_compat.GuardCompatibleDescr = AbstractFailDescr
+            self.check_rewrite("""
+                [p0]
+                guard_compatible(p0, ConstPtr(myR1)) []
+                guard_compatible(p0, ConstPtr(myR1)) []
+                jump()
+            """, """
+                [p0]
+                p1 = load_from_gc_table(1)
+                guard_compatible(p0, p1) []
+                guard_compatible(p0, p1) []
+                jump()
+            """)
+        finally:
+            guard_compat.GuardCompatibleDescr = org_GuardCompatibleDescr
+        for i, expected in enumerate([
+                'struct BACKEND_CHOICES ',
+                'struct R ',
+                'BasicFailDescr',
+                'struct BACKEND_CHOICES ',
+                'BasicFailDescr']):
             x = self.gcrefs[i]
-            lltype.cast_opaque_ptr(lltype.Ptr(guard_compat.BACKEND_CHOICES), x)
+            assert expected in repr(x._obj.container)

From pypy.commits at gmail.com  Fri Feb 10 11:54:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 08:54:23 -0800 (PST)
Subject: [pypy-commit] pypy guard-compatible: fix at least for
 backend/llsupport/test/test_zrpy_gc_direct.py
Message-ID: <589df03f.8ba4df0a.ebc3b.23e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: guard-compatible
Changeset: r90039:583cbba30cdb
Date: 2017-02-10 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/583cbba30cdb/

Log:	fix at least for backend/llsupport/test/test_zrpy_gc_direct.py

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1121,6 +1121,7 @@
 
     _compatibility_conditions = None
     other_compat_conditions = []
+    fallback_jump_target = 0
 
     def __init__(self):
         # XXX think about what is being kept alive here
@@ -1134,7 +1135,8 @@
         # to decide on
         # XXX it would be better to patch the guard properly in the backend,
         # but later
-        self.fallback_jump_target = 0
+        #self.fallback_jump_target = 0 (set on the class, annotator trick in
+        #                            case we don't see any GuardCompatibleDescr)
         # the next two attributes are for tracking where the guarded value came
         # from
         self.source_failarg_index = -1

From pypy.commits at gmail.com  Fri Feb 10 12:05:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 10 Feb 2017 09:05:39 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <589df2e3.7d96df0a.7e5ce.2767@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r854:2669787ce718
Date: 2017-02-10 11:18 +0100
http://bitbucket.org/pypy/pypy.org/changeset/2669787ce718/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66589 of $105000 (63.4%)
+   $66591 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2417 left
+    we have $2418 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $59040 of $80000 (73.8%)
+   $59050 of $80000 (73.8%)
    <div id="progressbar">
    </div>
 
@@ -29,7 +29,7 @@
   
   <!--
   Current status:
-    we have $48229 left
+    we have $48238 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Feb 10 14:42:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 Feb 2017 11:42:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use CPython implemntations for
 PyUnicode_GetSize, PyUnicode_GetLength
Message-ID: <589e17a1.48301c0a.13a9.017f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90040:8360e512f159
Date: 2017-02-10 19:41 +0000
http://bitbucket.org/pypy/pypy/changeset/8360e512f159/

Log:	Use CPython implemntations for PyUnicode_GetSize,
	PyUnicode_GetLength

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -563,6 +563,7 @@
     'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords',
     'PyArg_VaParse', 'PyArg_VaParseTupleAndKeywords', '_PyArg_NoKeywords',
     'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', 'PyUnicode_AsWideCharString',
+    'PyUnicode_GetSize', 'PyUnicode_GetLength',
     'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant',
     'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack',
     '_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT',
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -217,6 +217,19 @@
 
 /* === Public API ========================================================= */
 
+/* Get the length of the Unicode object. */
+
+PyAPI_FUNC(Py_ssize_t) PyUnicode_GetLength(
+    PyObject *unicode
+);
+
+/* Get the number of Py_UNICODE units in the
+   string representation. */
+
+PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(
+    PyObject *unicode           /* Unicode object */
+    );
+
 PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(
     const char *format,   /* ASCII-encoded string  */
     va_list vargs
diff --git a/pypy/module/cpyext/src/unicodeobject.c b/pypy/module/cpyext/src/unicodeobject.c
--- a/pypy/module/cpyext/src/unicodeobject.c
+++ b/pypy/module/cpyext/src/unicodeobject.c
@@ -551,3 +551,27 @@
     return buffer;
 }
 
+Py_ssize_t
+PyUnicode_GetSize(PyObject *unicode)
+{
+    if (!PyUnicode_Check(unicode)) {
+        PyErr_BadArgument();
+        goto onError;
+    }
+    return PyUnicode_GET_SIZE(unicode);
+
+  onError:
+    return -1;
+}
+
+Py_ssize_t
+PyUnicode_GetLength(PyObject *unicode)
+{
+    if (!PyUnicode_Check(unicode)) {
+        PyErr_BadArgument();
+        return -1;
+    }
+    if (PyUnicode_READY(unicode) == -1)
+        return -1;
+    return PyUnicode_GET_LENGTH(unicode);
+}
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -68,6 +68,16 @@
         assert module.test_is_unicode(u"")
         assert not module.test_is_unicode(())
 
+    def test_strlen(self):
+        module = self.import_extension('foo', [
+            ('strlen', "METH_O",
+             """
+                PyObject* s = PyObject_Str(args);
+                return PyLong_FromLong(PyUnicode_GetLength(s));
+             """)])
+        print(module.strlen(True))
+        assert module.strlen(True) == 4
+
     def test_unicode_buffer_init(self):
         module = self.import_extension('foo', [
             ("getunicode", "METH_NOARGS",
@@ -213,9 +223,6 @@
 
 class TestUnicode(BaseApiTest):
     def test_unicodeobject(self, space):
-        assert PyUnicode_GetSize(space, space.wrap(u'sp&#228;m')) == 4
-        unichar = rffi.sizeof(Py_UNICODE)
-
         encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, ))
         w_default_encoding = space.call_function(
             space.sys.get('getdefaultencoding')
@@ -627,7 +634,7 @@
 
         assert space.eq_w(w_y, space.wrap(u"abcd"))
 
-        size = PyUnicode_GetSize(space, w_x)
+        size = get_wsize(as_pyobj(space, w_x))
         Py_UNICODE_COPY(space, target_chunk, x_chunk, size)
         w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size))
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -273,6 +273,7 @@
             w_obj._value, len(w_obj._value), errors='strict'))
         set_data(py_obj, cts.cast('void*', ucs1_data))
         set_kind(py_obj, _1BYTE_KIND)
+        set_len(py_obj, get_wsize(py_obj))
         if maxchar < 128:
             set_ascii(py_obj, 1)
             set_utf8(py_obj, cts.cast('char*', get_data(py_obj)))
@@ -328,28 +329,6 @@
         set_utf8(ref, rffi.str2charp(s))
     return get_utf8(ref)
 
- at cpython_api([PyObject], Py_ssize_t, error=-1)
-def PyUnicode_GetSize(space, ref):
-    """Return the size of the deprecated Py_UNICODE representation, in code
-    units (this includes surrogate pairs as 2 units).
-
-    Please migrate to using PyUnicode_GetLength().
-    """
-    if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_unicode:
-        return get_wsize(ref)
-    else:
-        w_obj = from_ref(space, ref)
-        return space.len_w(w_obj)
-
- at cpython_api([PyObject], Py_ssize_t, error=-1)
-def PyUnicode_GetLength(space, unicode):
-    """Return the length of the Unicode object, in code points."""
-    # XXX: this is a stub
-    if not PyUnicode_Check(space, unicode):
-        PyErr_BadArgument(space)
-    #PyUnicode_READY(w_unicode)
-    return get_len(unicode)
-
 @cpython_api([PyObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1)
 def PyUnicode_AsWideChar(space, ref, buf, size):
     """Copy the Unicode object contents into the wchar_t buffer w.  At most

From pypy.commits at gmail.com  Fri Feb 10 15:19:40 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 10 Feb 2017 12:19:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test and fix for PyLong_AsLong
Message-ID: <589e205c.2687df0a.3e8d6.6800@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90041:7d3aa9ec294f
Date: 2017-02-10 20:19 +0000
http://bitbucket.org/pypy/pypy/changeset/7d3aa9ec294f/

Log:	Test and fix for PyLong_AsLong

diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -73,7 +73,7 @@
     Return a C long representation of the contents of pylong.  If
     pylong is greater than LONG_MAX, an OverflowError is raised
     and -1 will be returned."""
-    return space.int_w(w_long)
+    return space.int_w(space.int(w_long))
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
 def PyLong_AsSsize_t(space, w_long):
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -269,6 +269,20 @@
         # A string with arabic digits. 'BAD' is after the 6th character.
         assert module.from_unicode(u'  1\u0662\u0663\u0664BAD') == (1234, 4660)
 
+    def test_aslong(self):
+        module = self.import_extension('foo', [
+            ("as_long", "METH_O",
+             """
+                long n = PyLong_AsLong(args);
+                if (n == -1 && PyErr_Occurred()) {
+                    return NULL;
+                }
+                return PyLong_FromLong(n);
+             """)])
+        assert module.as_long(123) == 123
+        assert module.as_long(-1) == -1
+        assert module.as_long(1.23) == 1
+
     def test_strtol(self):
         module = self.import_extension('foo', [
             ("from_str", "METH_NOARGS",

From pypy.commits at gmail.com  Sat Feb 11 06:25:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 03:25:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: forgot to update this
Message-ID: <589ef4c0.dea2df0a.7850d.11fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90042:a1af975cae9c
Date: 2017-02-11 12:25 +0100
http://bitbucket.org/pypy/pypy/changeset/a1af975cae9c/

Log:	forgot to update this

diff --git a/lib-python/stdlib-version.txt b/lib-python/stdlib-version.txt
--- a/lib-python/stdlib-version.txt
+++ b/lib-python/stdlib-version.txt
@@ -6,4 +6,4 @@
     2.7::
         a06454b1afa1 (2.7) v2.7.13
     3::
-        37a07cee5969 (3.5) v3.5.1
+        1880cb95a742 (3.5) v3.5.3

From pypy.commits at gmail.com  Sat Feb 11 08:55:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 05:55:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <589f17e4.85df1c0a.2f5b6.3b9d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90043:6f94d4627b17
Date: 2017-02-11 14:36 +0100
http://bitbucket.org/pypy/pypy/changeset/6f94d4627b17/

Log:	fix

diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -329,7 +329,14 @@
 
     @classmethod
     def from_float(cls, value):
-        return cls._from_float(value, getcontext(), exact=True)
+        # note: if 'cls' is a subclass of Decimal and 'value' is an int,
+        # this will call cls(Decimal('42')) whereas _pydecimal.py will
+        # call cls(42).  This is like CPython's _decimal module.
+        result = cls._from_float(value, getcontext(), exact=True)
+        if cls is Decimal:
+            return result
+        else:
+            return cls(result)
 
     @classmethod
     def _from_float(cls, value, context, exact=True):

From pypy.commits at gmail.com  Sat Feb 11 08:55:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 05:55:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix for
 test_internal_use_of_overridden_methods
Message-ID: <589f17e6.11341c0a.92d3a.3d06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90044:7e5dd6bf5bd1
Date: 2017-02-11 14:55 +0100
http://bitbucket.org/pypy/pypy/changeset/7e5dd6bf5bd1/

Log:	fix for test_internal_use_of_overridden_methods

diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -240,7 +240,7 @@
 
     @classmethod
     def _from_str(cls, value, context, exact=True, strip=True):
-        s = value.encode('ascii', '_decimal_encode')
+        s = str.encode(value, 'ascii', '_decimal_encode')
         if b'\0' in s:
             s = b''  # empty string triggers ConversionSyntax.
         if strip:
@@ -256,6 +256,7 @@
 
     @classmethod
     def _from_int(cls, value, context, exact=True):
+        value = int(value)     # in case it's a subclass of 'int'
         self = cls._new_empty()
         with _CatchConversions(self._mpd, context, exact) as (ctx, status_ptr):
             size = (((value|1).bit_length() + 15) // 16) + 5
@@ -332,6 +333,8 @@
         # note: if 'cls' is a subclass of Decimal and 'value' is an int,
         # this will call cls(Decimal('42')) whereas _pydecimal.py will
         # call cls(42).  This is like CPython's _decimal module.
+        if not isinstance(value, (int, float)):
+            raise TypeError("argument must be int of float")
         result = cls._from_float(value, getcontext(), exact=True)
         if cls is Decimal:
             return result
@@ -342,6 +345,7 @@
     def _from_float(cls, value, context, exact=True):
         if isinstance(value, int):
             return cls._from_int(value, context, exact=exact)
+        value = float(value)    # in case it's a subclass of 'float'
         sign = 0 if _math.copysign(1.0, value) == 1.0 else 1
 
         if _math.isnan(value):

From pypy.commits at gmail.com  Sat Feb 11 09:00:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 06:00:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: check_syntax_error(): ignore the offset
 on pypy,
 it can often be slightly different than on CPython and I think it's fine
Message-ID: <589f1910.6483df0a.fc170.3293@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90045:7e5648cd878f
Date: 2017-02-11 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/7e5648cd878f/

Log:	check_syntax_error(): ignore the offset on pypy, it can often be
	slightly different than on CPython and I think it's fine

diff --git a/lib-python/3/test/support/__init__.py b/lib-python/3/test/support/__init__.py
--- a/lib-python/3/test/support/__init__.py
+++ b/lib-python/3/test/support/__init__.py
@@ -1049,7 +1049,9 @@
     if lineno is not None:
         testcase.assertEqual(err.lineno, lineno)
     testcase.assertIsNotNone(err.offset)
-    if offset is not None:
+    # Don't check the offset on PyPy, it can often be slightly different
+    # than on CPython
+    if offset is not None and check_impl_detail():
         testcase.assertEqual(err.offset, offset)
 
 def open_urlresource(url, *args, **kw):

From pypy.commits at gmail.com  Sat Feb 11 09:47:30 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 06:47:30 -0800 (PST)
Subject: [pypy-commit] pypy default: Minor optimization
Message-ID: <589f2402.90b1df0a.4f04b.3f36@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90046:d67b8ad83f22
Date: 2017-02-11 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/d67b8ad83f22/

Log:	Minor optimization

diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -722,14 +722,13 @@
         self.space = space
         self.saved_w = []
         self.w_iterable = space.iter(w_iterable)
-        self.index = 0
-        self.exhausted = False
+        self.index = 0    # 0 during the first iteration; > 0 afterwards
 
     def iter_w(self):
         return self.space.wrap(self)
 
     def next_w(self):
-        if self.exhausted:
+        if self.index > 0:
             if not self.saved_w:
                 raise OperationError(self.space.w_StopIteration, self.space.w_None)
             try:
@@ -744,15 +743,13 @@
                 w_obj = self.space.next(self.w_iterable)
             except OperationError as e:
                 if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
+                    self.index = 1
                     if not self.saved_w:
                         raise
-                    self.index = 1
                     w_obj = self.saved_w[0]
                 else:
                     raise
             else:
-                self.index += 1
                 self.saved_w.append(w_obj)
         return w_obj
 

From pypy.commits at gmail.com  Sat Feb 11 09:47:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 06:47:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <589f2404.45aadf0a.e0fe9.4a9b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90047:88164726e74b
Date: 2017-02-11 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/88164726e74b/

Log:	hg merge default

diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -462,7 +462,12 @@
         field or array item in the structure or array, recursively in
         case of nested structures.
         """
-        ctype = self._backend.typeof(cdata)
+        try:
+            ctype = self._backend.typeof(cdata)
+        except TypeError:
+            if '__addressof__' in type(cdata).__dict__:
+                return type(cdata).__addressof__(cdata, *fields_or_indexes)
+            raise
         if fields_or_indexes:
             ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
         else:
@@ -775,10 +780,7 @@
         key = 'function ' + name
         tp, _ = ffi._parser._declarations[key]
         BType = ffi._get_cached_btype(tp)
-        try:
-            value = backendlib.load_function(BType, name)
-        except KeyError as e:
-            raise AttributeError('%s: %s' % (name, e))
+        value = backendlib.load_function(BType, name)
         library.__dict__[name] = value
     #
     def accessor_variable(name):
@@ -791,6 +793,21 @@
             lambda self: read_variable(BType, name),
             lambda self, value: write_variable(BType, name, value)))
     #
+    def addressof_var(name):
+        try:
+            return addr_variables[name]
+        except KeyError:
+            with ffi._lock:
+                if name not in addr_variables:
+                    key = 'variable ' + name
+                    tp, _ = ffi._parser._declarations[key]
+                    BType = ffi._get_cached_btype(tp)
+                    if BType.kind != 'array':
+                        BType = model.pointer_cache(ffi, BType)
+                    p = backendlib.load_function(BType, name)
+                    addr_variables[name] = p
+            return addr_variables[name]
+    #
     def accessor_constant(name):
         raise NotImplementedError("non-integer constant '%s' cannot be "
                                   "accessed from a dlopen() library" % (name,))
@@ -800,6 +817,7 @@
     #
     accessors = {}
     accessors_version = [False]
+    addr_variables = {}
     #
     def update_accessors():
         if accessors_version[0] is ffi._cdef_version:
@@ -850,6 +868,18 @@
             with ffi._lock:
                 update_accessors()
                 return accessors.keys()
+        def __addressof__(self, name):
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            make_accessor(name)
+            if name in library.__dict__:
+                return library.__dict__[name]
+            if name in FFILibrary.__dict__:
+                return addressof_var(name)
+            raise AttributeError("cffi library has no function or "
+                                 "global variable named '%s'" % (name,))
     #
     if libname is not None:
         try:
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.11
+Version: 0.4.12
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -2,7 +2,7 @@
 import __pypy__
 import _continuation
 
-__version__ = "0.4.11"
+__version__ = "0.4.12"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -38,6 +38,21 @@
 .. _mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 
 
+Explicit typing in RPython
+--------------------------
+
+RPython is mostly based around type inference, but there are many cases where
+specifying types explicitly is useful. We would like to be able to optionally 
+specify the exact types of the arguments to any function. We already have
+solutions in that space, ``@rpython.rlib.objectmodel.enforceargs`` and
+``@rpython.rlib.signature.signature``, but they are inconvenient and limited. 
+For instance, they do not easily allow to express the type "dict with ints as
+keys and lists of instances of Foo as values".
+
+Additionally, we would like to be able to specify the types of instance
+attributes. Unlike the function case, this is likely to require some
+refactoring of the annotator. 
+
 Make bytearray type fast
 ------------------------
 
@@ -176,28 +191,33 @@
 * `hg`
 
 
-Optimising cpyext (CPython C-API compatibility layer)
------------------------------------------------------
-
-A lot of work has gone into PyPy's implementation of CPython's C-API over
-the last years to let it reach a practical level of compatibility, so that
-C extensions for CPython work on PyPy without major rewrites. However,
-there are still many edges and corner cases where it misbehaves.
-
-The objective of this project is to fix bugs in cpyext and to optimise
-several performance critical parts of it, such as the reference counting
-support and other heavily used C-API functions. The net result would be to
-have CPython extensions run much faster on PyPy than they currently do, or
-to make them work at all if they currently don't. A part of this work would
-be to get cpyext into a shape where it supports running Cython generated
-extensions.
-
 ======================================
 Make more python modules pypy-friendly
 ======================================
 
-Work has been started on a few popular python packages. Here is a partial
-list of good work that needs to be finished:
+A lot of work has gone into PyPy's implementation of CPython's C-API, cpyext,
+over the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However, there
+are still many edges and corner cases where it misbehaves.
+
+For any popular extension that does not already advertise full PyPy
+compatibility, it would thus be useful to take a close look at it in order to
+make it fully compatible with PyPy. The general process is something like:
+
+* Run the extension's tests on PyPy and look at the test failures.
+* Some of the failures may be solved by identifying cases where the extension
+  relies on undocumented or internal details of CPython, and rewriting the
+  relevant code to follow documented best practices. Open issues and send pull
+  requests as appropriate given the extension's development process.
+* Other failures may highlight incompatibilities between cpyext and CPython.
+  Please report them to us and try to fix them.
+* Run benchmarks, either provided by the extension developers or created by
+  you. Any case where PyPy is significantly slower than CPython is to be
+  considered a bug and solved as above.
+
+Alternatively, an approach we used to recommend was to rewrite C extensions
+using more pypy-friendly technologies, e.g. cffi. Here is a partial list of
+good work that needs to be finished:
 
 **matplotlib** https://github.com/mattip/matplotlib
 
@@ -223,5 +243,3 @@
 **pyopengl** https://bitbucket.org/duangle/pyopengl-cffi
 
     Status: unknown
-
-
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -40,25 +40,22 @@
 
     @unwrap_spec(w_ctype=W_CType, name=str)
     def load_function(self, w_ctype, name):
-        from pypy.module._cffi_backend import ctypefunc, ctypeptr, ctypevoid
+        from pypy.module._cffi_backend import ctypeptr, ctypearray
         space = self.space
         #
-        ok = False
-        if isinstance(w_ctype, ctypefunc.W_CTypeFunc):
-            ok = True
-        if (isinstance(w_ctype, ctypeptr.W_CTypePointer) and
-            isinstance(w_ctype.ctitem, ctypevoid.W_CTypeVoid)):
-            ok = True
-        if not ok:
+        if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray):
             raise oefmt(space.w_TypeError,
-                        "function cdata expected, got '%s'", w_ctype.name)
+                        "function or pointer or array cdata expected, got '%s'",
+                        w_ctype.name)
         #
         try:
             cdata = dlsym(self.handle, name)
         except KeyError:
-            raise oefmt(space.w_KeyError,
-                        "function '%s' not found in library '%s'",
+            raise oefmt(space.w_AttributeError,
+                        "function/symbol '%s' not found in library '%s'",
                         name, self.name)
+        if isinstance(w_ctype, ctypearray.W_CTypeArray) and w_ctype.length < 0:
+            w_ctype = w_ctype.ctptr
         return W_CData(space, rffi.cast(rffi.CCHARP, cdata), w_ctype)
 
     @unwrap_spec(w_ctype=W_CType, name=str)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -365,7 +365,7 @@
     x = find_and_load_library(None)
     BVoidP = new_pointer_type(new_void_type())
     assert x.load_function(BVoidP, 'strcpy')
-    py.test.raises(KeyError, x.load_function,
+    py.test.raises(AttributeError, x.load_function,
                    BVoidP, 'xxx_this_function_does_not_exist')
     # the next one is from 'libm', not 'libc', but we assume
     # that it is already loaded too, so it should work
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -670,14 +670,13 @@
         self.space = space
         self.saved_w = []
         self.w_iterable = space.iter(w_iterable)
-        self.index = 0
-        self.exhausted = False
+        self.index = 0    # 0 during the first iteration; > 0 afterwards
 
     def iter_w(self):
         return self.space.wrap(self)
 
     def next_w(self):
-        if self.exhausted:
+        if self.index > 0:
             if not self.saved_w:
                 raise OperationError(self.space.w_StopIteration, self.space.w_None)
             try:
@@ -692,26 +691,27 @@
                 w_obj = self.space.next(self.w_iterable)
             except OperationError as e:
                 if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
+                    self.index = 1
                     if not self.saved_w:
                         raise
-                    self.index = 1
                     w_obj = self.saved_w[0]
                 else:
                     raise
             else:
-                self.index += 1
                 self.saved_w.append(w_obj)
         return w_obj
 
     def descr_reduce(self, space):
+        # reduces differently than CPython 3.5.  Unsure if it is a
+        # problem.  To be on the safe side, keep three arguments for
+        # __setstate__; CPython takes two.
         return space.newtuple([
             space.type(self),
             space.newtuple([self.w_iterable]),
             space.newtuple([
                 space.newlist(self.saved_w),
                 space.wrap(self.index),
-                space.wrap(self.exhausted),
+                space.wrap(self.index > 0),
             ]),
         ])
 
@@ -719,7 +719,7 @@
         w_saved, w_index, w_exhausted = space.unpackiterable(w_state, 3)
         self.saved_w = space.unpackiterable(w_saved)
         self.index = space.int_w(w_index)
-        self.exhausted = space.bool_w(w_exhausted)
+        # w_exhausted ignored
 
 
 def W_Cycle___new__(space, w_subtype, w_iterable):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1227,6 +1227,26 @@
         assert list(a)[:1000] + [0] * (len(a)-1000) == list(b)
         f.close()
 
+    def test_ffi_buffer_comparisons(self):
+        ffi = FFI(backend=self.Backend())
+        ba = bytearray(range(100, 110))
+        assert ba == memoryview(ba)    # justification for the following
+        a = ffi.new("uint8_t[]", list(ba))
+        c = ffi.new("uint8_t[]", [99] + list(ba))
+        try:
+            b_full = ffi.buffer(a)
+            b_short = ffi.buffer(a, 3)
+            b_mid = ffi.buffer(a, 6)
+            b_other = ffi.buffer(c, 6)
+        except NotImplementedError as e:
+            py.test.skip(str(e))
+        else:
+            content = b_full[:]
+            assert content == b_full == ba
+            assert b_other < b_short < b_mid < b_full
+            assert ba > b_mid > ba[0:2]
+            assert b_short != ba[1:4]
+
     def test_array_in_struct(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct foo_s { int len; short data[5]; };")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -193,9 +193,12 @@
         setattr(s, name, value)
         assert getattr(s, name) == value
         raw1 = ffi.buffer(s)[:]
+        buff1 = ffi.buffer(s)
         t = lib.try_with_value(fnames.index(name), value)
         raw2 = ffi.buffer(t, len(raw1))[:]
         assert raw1 == raw2
+        buff2 = ffi.buffer(t, len(buff1))
+        assert buff1 == buff2
 
     def test_bitfield_basic(self):
         self.check("int a; int b:9; int c:20; int y;", 8, 4, 12)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -283,3 +283,21 @@
             assert ret.right == ownlib.right
             assert ret.top == ownlib.top
             assert ret.bottom == ownlib.bottom
+
+    def test_addressof_lib(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("long left; int test_getting_errno(void);")
+        lib = ffi.dlopen(self.module)
+        lib.left = 123456
+        p = ffi.addressof(lib, "left")
+        assert ffi.typeof(p) == ffi.typeof("long *")
+        assert p[0] == 123456
+        p[0] += 1
+        assert lib.left == 123457
+        pfn = ffi.addressof(lib, "test_getting_errno")
+        assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
+        assert pfn == lib.test_getting_errno
diff --git a/rpython/translator/c/src/stacklet/stacklet.c b/rpython/translator/c/src/stacklet/stacklet.c
--- a/rpython/translator/c/src/stacklet/stacklet.c
+++ b/rpython/translator/c/src/stacklet/stacklet.c
@@ -5,8 +5,8 @@
 #include "src/stacklet/stacklet.h"
 
 #include <stddef.h>
-#include <assert.h>
 #include <string.h>
+#include <stdio.h>
 
 /************************************************************
  * platform specific code
@@ -34,7 +34,7 @@
 /************************************************************/
 
 struct stacklet_s {
-    /* The portion of the real stack claimed by this paused tealet. */
+    /* The portion of the real stack claimed by this paused stacklet. */
     char *stack_start;                /* the "near" end of the stack */
     char *stack_stop;                 /* the "far" end of the stack */
 
@@ -69,6 +69,19 @@
     struct stacklet_s *g_target;
 };
 
+#define _check(x)  do { if (!(x)) _check_failed(#x); } while (0)
+
+static void _check_failed(const char *check)
+{
+    fprintf(stderr, "FATAL: stacklet: %s failed\n", check);
+    abort();
+}
+
+static void check_valid(struct stacklet_s *g)
+{
+    _check(g->stack_saved >= 0);
+}
+
 /***************************************************************/
 
 static void g_save(struct stacklet_s* g, char* stop
@@ -96,7 +109,8 @@
      */
     ptrdiff_t sz1 = g->stack_saved;
     ptrdiff_t sz2 = stop - g->stack_start;
-    assert(stop <= g->stack_stop);
+    check_valid(g);
+    _check(stop <= g->stack_stop);
 
     if (sz2 > sz1) {
         char *c = (char *)(g + 1);
@@ -146,11 +160,13 @@
 {
     struct stacklet_s *current = thrd->g_stack_chain_head;
     char *target_stop = g_target->stack_stop;
+    check_valid(g_target);
 
-    /* save and unlink tealets that are completely within
+    /* save and unlink stacklets that are completely within
        the area to clear. */
     while (current != NULL && current->stack_stop <= target_stop) {
         struct stacklet_s *prev = current->stack_prev;
+        check_valid(current);
         current->stack_prev = NULL;
         if (current != g_target) {
             /* don't bother saving away g_target, because
@@ -222,14 +238,16 @@
     struct stacklet_thread_s *thrd = (struct stacklet_thread_s *)rawthrd;
     struct stacklet_s *g = thrd->g_target;
     ptrdiff_t stack_saved = g->stack_saved;
+    check_valid(g);
 
-    assert(new_stack_pointer == g->stack_start);
+    _check(new_stack_pointer == g->stack_start);
 #if STACK_DIRECTION == 0
     memcpy(g->stack_start, g+1, stack_saved);
 #else
     memcpy(g->stack_start - stack_saved, g+1, stack_saved);
 #endif
     thrd->g_current_stack_stop = g->stack_stop;
+    g->stack_saved = -13;   /* debugging */
     free(g);
     return EMPTY_STACKLET_HANDLE;
 }
@@ -250,10 +268,11 @@
         result = run(thrd->g_source, run_arg);
 
         /* Then switch to 'result'. */
+        check_valid(result);
         thrd->g_target = result;
         _stacklet_switchstack(g_destroy_state, g_restore_state, thrd);
 
-        assert(!"stacklet: we should not return here");
+        _check_failed("we should not return here");
         abort();
     }
     /* The second time it returns. */
@@ -287,7 +306,7 @@
                              stacklet_run_fn run, void *run_arg)
 {
     long stackmarker;
-    assert((char *)NULL < (char *)&stackmarker);
+    _check((char *)NULL < (char *)&stackmarker);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
@@ -300,6 +319,7 @@
 {
     long stackmarker;
     stacklet_thread_handle thrd = target->stack_thrd;
+    check_valid(target);
     if (thrd->g_current_stack_stop <= (char *)&stackmarker)
         thrd->g_current_stack_stop = ((char *)&stackmarker) + 1;
 
@@ -310,6 +330,7 @@
 
 void stacklet_destroy(stacklet_handle target)
 {
+    check_valid(target);
     if (target->stack_prev != NULL) {
         /* 'target' appears to be in the chained list 'unsaved_stack',
            so remove it from there.  Note that if 'thrd' was already
@@ -319,12 +340,15 @@
            we don't even read 'stack_thrd', already deallocated. */
         stacklet_thread_handle thrd = target->stack_thrd;
         struct stacklet_s **pp = &thrd->g_stack_chain_head;
-        for (; *pp != NULL; pp = &(*pp)->stack_prev)
+        for (; *pp != NULL; pp = &(*pp)->stack_prev) {
+            check_valid(*pp);
             if (*pp == target) {
                 *pp = target->stack_prev;
                 break;
             }
+        }
     }
+    target->stack_saved = -11;   /* debugging */
     free(target);
 }
 
@@ -334,6 +358,7 @@
   long delta;
   if (context == NULL)
     return ptr;
+  check_valid(context);
   delta = p - context->stack_start;
   if (((unsigned long)delta) < ((unsigned long)context->stack_saved)) {
       /* a pointer to a saved away word */
@@ -345,8 +370,8 @@
       /* out-of-stack pointer!  it's only ok if we are the main stacklet
          and we are reading past the end, because the main stacklet's
          stack stop is not exactly known. */
-      assert(delta >= 0);
-      assert(((long)context->stack_stop) & 1);
+      _check(delta >= 0);
+      _check(((long)context->stack_stop) & 1);
   }
   return ptr;
 }

From pypy.commits at gmail.com  Sat Feb 11 09:47:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 06:47:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip these two tests
Message-ID: <589f2406.c1601c0a.de77f.45ab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90048:2abc31ad4db0
Date: 2017-02-11 15:41 +0100
http://bitbucket.org/pypy/pypy/changeset/2abc31ad4db0/

Log:	skip these two tests

diff --git a/lib-python/3/test/test_itertools.py b/lib-python/3/test/test_itertools.py
--- a/lib-python/3/test/test_itertools.py
+++ b/lib-python/3/test/test_itertools.py
@@ -184,6 +184,9 @@
         for proto in range(pickle.HIGHEST_PROTOCOL + 1):
             self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef'))
 
+    @support.impl_detail("XXX chain.__setstate__ does not do all checks"
+                         " on PyPy, will just complain later (but could"
+                         " be fixed if important)")
     def test_chain_setstate(self):
         self.assertRaises(TypeError, chain().__setstate__, ())
         self.assertRaises(TypeError, chain().__setstate__, [])
@@ -644,6 +647,8 @@
         for proto in range(pickle.HIGHEST_PROTOCOL + 1):
             self.pickletest(proto, cycle('abc'))
 
+    @support.impl_detail("XXX cycle.__reduce__ and __setstate__ differ"
+                         " on PyPy (but could be fixed if important)")
     def test_cycle_setstate(self):
         self.assertRaises(TypeError, cycle('').__setstate__, ())
         self.assertRaises(TypeError, cycle('').__setstate__, [])

From pypy.commits at gmail.com  Sat Feb 11 10:06:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 07:06:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix datetime.__hash__
Message-ID: <589f285d.44641c0a.48d56.edfc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90049:63cd193b7617
Date: 2017-02-11 16:05 +0100
http://bitbucket.org/pypy/pypy/changeset/63cd193b7617/

Log:	Fix datetime.__hash__

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -1145,16 +1145,17 @@
     def __hash__(self):
         """Hash."""
         if self._hashcode == -1:
-            # PyPy: uses the same algo as _datetimemodule.c, which
-            # unlike the pure Python version always relies on the
-            # nondeterministic hash on strings
-            temp1 = timedelta(hours=self._hour,
-                              minutes=self._minute,
-                              seconds=self._second,
-                              microseconds=self._microsecond)
-            tzoff = self.utcoffset()
-            if tzoff:  # not zero, not None
-                temp1 -= tzoff
+            # PyPy: uses an algo that, like _datetimemodule.c and
+            # unlike the pure Python version, always relies on the
+            # nondeterministic hash on strings.  Well, if we have no
+            # tzoff, that is.  If we have tzoff then CPython's hashes
+            # are again deterministic.  I have no clue why.  We'll go
+            # for now for also being nondeterministic in this case.
+            temp1 = '%d@%d@%d@%d@%s' % (self._hour,
+                                        self._minute,
+                                        self._second,
+                                        self._microsecond,
+                                        self.utcoffset())
             self._hashcode = hash(temp1)
         return self._hashcode
 

From pypy.commits at gmail.com  Sat Feb 11 10:58:40 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Sat, 11 Feb 2017 07:58:40 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: remove allocated libunwind info
 for dynamic code regions as soon as the loop token goes out of scope
Message-ID: <589f34b0.5190df0a.a8f13.590b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90050:fd79187556df
Date: 2017-02-11 16:58 +0100
http://bitbucket.org/pypy/pypy/changeset/fd79187556df/

Log:	remove allocated libunwind info for dynamic code regions as soon as
	the loop token goes out of scope

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -491,7 +491,6 @@
         if not we_are_translated():
             # Arguments should be unique
             assert len(set(inputargs)) == len(inputargs)
-
         self.setup(looptoken)
         if self.cpu.HAS_CODEMAP:
             self.codemap_builder.enter_portal_frame(jd_id, unique_id,
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -160,9 +160,10 @@
     jitcell_token.outermost_jitdriver_sd = jitdriver_sd
     return jitcell_token
 
-def record_loop_or_bridge(metainterp_sd, loop):
+def record_loop_or_bridge(metainterp_sd, loop, asminfo):
     """Do post-backend recordings and cleanups on 'loop'.
     """
+    from rpython.rlib.rvmprof import rvmprof
     # get the original jitcell token corresponding to jitcell form which
     # this trace starts
     original_jitcell_token = loop.original_jitcell_token
@@ -172,6 +173,11 @@
     wref = weakref.ref(original_jitcell_token)
     clt = original_jitcell_token.compiled_loop_token
     clt.loop_token_wref = wref
+
+    rvmprof.dyn_register_jit_page(original_jitcell_token, asminfo.asmaddr,
+                                  asminfo.asmaddr+asminfo.asmlen)
+
+
     for op in loop.operations:
         descr = op.getdescr()
         # not sure what descr.index is about
@@ -242,9 +248,9 @@
     if not we_are_translated():
         loop.check_consistency()
     jitcell_token.target_tokens = [target_token]
-    send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
-                         runtime_args, metainterp.box_names_memo)
-    record_loop_or_bridge(metainterp_sd, loop)
+    asminfo = send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd,
+                    loop, "loop", runtime_args, metainterp.box_names_memo)
+    record_loop_or_bridge(metainterp_sd, loop, asminfo)
     return target_token
 
 def compile_loop(metainterp, greenkey, start, inputargs, jumpargs,
@@ -337,9 +343,9 @@
                        loop_info.extra_before_label + [loop_info.label_op] + loop_ops)
     if not we_are_translated():
         loop.check_consistency()
-    send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
-                         inputargs, metainterp.box_names_memo)
-    record_loop_or_bridge(metainterp_sd, loop)
+    asminfo = send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd,
+                    loop, "loop", inputargs, metainterp.box_names_memo)
+    record_loop_or_bridge(metainterp_sd, loop, asminfo)
     loop_info.post_loop_compilation(loop, jitdriver_sd, metainterp, jitcell_token)
     return start_descr
 
@@ -414,9 +420,9 @@
         loop.quasi_immutable_deps = quasi_immutable_deps
 
     target_token = loop.operations[-1].getdescr()
-    resumekey.compile_and_attach(metainterp, loop, inputargs)
+    asminfo = resumekey.compile_and_attach(metainterp, loop, inputargs)
 
-    record_loop_or_bridge(metainterp_sd, loop)
+    record_loop_or_bridge(metainterp_sd, loop, asminfo)
     return target_token
 
 def get_box_replacement(op, allow_none=False):
@@ -506,11 +512,6 @@
                                           jd_id=jd_id, unique_id=unique_id,
                                           log=log, name=name, logger=metainterp_sd.jitlog)
 
-    vmprof = metainterp_sd.vmprof
-    if vmprof:
-        vmprof.dyn_register_jit_page(asminfo.asmaddr,
-                                     asminfo.asmaddr+asminfo.asmlen, 1)
-
     return asminfo
 
 def do_compile_bridge(metainterp_sd, faildescr, inputargs, operations,
@@ -524,10 +525,6 @@
     asminfo = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations,
                                             original_loop_token, log=log,
                                             logger=metainterp_sd.jitlog)
-    vmprof = metainterp_sd.vmprof
-    if vmprof:
-        vmprof.dyn_register_jit_page(asminfo.asmaddr,
-                                     asminfo.asmaddr+asminfo.asmlen, 0)
     return asminfo
 
 def forget_optimization_info(lst, reset_values=False):
@@ -598,6 +595,8 @@
     if metainterp_sd.warmrunnerdesc is not None:    # for tests
         metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(original_jitcell_token)
 
+    return asminfo
+
 def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs,
                            operations, original_loop_token, memo):
     forget_optimization_info(operations)
@@ -843,10 +842,10 @@
             self._debug_subinputargs = new_loop.inputargs
             self._debug_suboperations = new_loop.operations
         propagate_original_jitcell_token(new_loop)
-        send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata,
-                               self, inputargs, new_loop.operations,
-                               new_loop.original_jitcell_token,
-                               metainterp.box_names_memo)
+        return send_bridge_to_backend(metainterp.jitdriver_sd,
+                    metainterp.staticdata, self, inputargs,
+                    new_loop.operations, new_loop.original_jitcell_token,
+                    metainterp.box_names_memo)
 
     def make_a_counter_per_value(self, guard_value_op, index):
         assert guard_value_op.getopnum() == rop.GUARD_VALUE
@@ -1047,13 +1046,14 @@
         jitdriver_sd = metainterp.jitdriver_sd
         new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd)
         propagate_original_jitcell_token(new_loop)
-        send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd,
-                             metainterp_sd, new_loop, "entry bridge",
-                             orig_inputargs, metainterp.box_names_memo)
+        asminfo = send_loop_to_backend(self.original_greenkey,
+                     metainterp.jitdriver_sd, metainterp_sd, new_loop,
+                     "entry bridge", orig_inputargs, metainterp.box_names_memo)
         # send the new_loop to warmspot.py, to be called directly the next time
         jitdriver_sd.warmstate.attach_procedure_to_interp(
             self.original_greenkey, jitcell_token)
         metainterp_sd.stats.add_jitcell_token(jitcell_token)
+        return asminfo
 
 
 def compile_trace(metainterp, resumekey, runtime_boxes):
@@ -1115,8 +1115,8 @@
     if info.final():
         new_trace.inputargs = info.inputargs
         target_token = new_trace.operations[-1].getdescr()
-        resumekey.compile_and_attach(metainterp, new_trace, inputargs)
-        record_loop_or_bridge(metainterp_sd, new_trace)
+        asminfo = resumekey.compile_and_attach(metainterp, new_trace, inputargs)
+        record_loop_or_bridge(metainterp_sd, new_trace, asminfo)
         return target_token
     new_trace.inputargs = info.renamed_inputargs
     metainterp.retrace_needed(new_trace, info)
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -419,6 +419,14 @@
     def __init__(self):
         # For memory management of assembled loops
         self._keepalive_jitcell_tokens = {}      # set of other JitCellToken
+        self._rvmprof_references = []
+
+    def rvmprof_register(self, ref):
+        """ Call this method for every loop or bridge that hangs on this
+            token. Otherwise the information tracked by libunwind will
+            not be freed.
+        """
+        self._rvmprof_references.append(ref)
 
     def record_jump_to(self, jitcell_token):
         assert isinstance(jitcell_token, JitCellToken)
diff --git a/rpython/jit/metainterp/memmgr.py b/rpython/jit/metainterp/memmgr.py
--- a/rpython/jit/metainterp/memmgr.py
+++ b/rpython/jit/metainterp/memmgr.py
@@ -2,6 +2,7 @@
 from rpython.rlib.rarithmetic import r_int64
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rvmprof import rvmprof
 
 #
 # Logic to decide which loops are old and not used any more.
@@ -70,6 +71,7 @@
         for looptoken in self.alive_loops.keys():
             if (0 <= looptoken.generation < max_generation or
                 looptoken.invalidated):
+                rvmprof.vmp_dyn_cancel(looptoken)
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -54,6 +54,7 @@
         SHARED.join('machine.c'),
         SHARED.join('symboltable.c'),
         SHARED.join('vmp_stack.c'),
+        SHARED.join('vmp_dynamic.c'),
         # udis86
         SHARED.join('libudis86/decode.c'),
         SHARED.join('libudis86/itab.c'),
@@ -96,6 +97,19 @@
                                   lltype.Signed, compilation_info=eci,
                                   _nowrapper=True)
 
+    vmp_dyn_register_jit_page = rffi.llexternal("vmp_dyn_register_jit_page",
+                                  [lltype.Signed, lltype.Signed, rffi.CCHARP],
+                                  rffi.INT, compilation_info=eci,
+                                  _nowrapper=True)
+
+    vmp_dyn_cancel = rffi.llexternal("vmp_dyn_cancel", [rffi.INT],
+                                  lltype.Void, compilation_info=eci,
+                                  _nowrapper=True)
+
+    vmp_dyn_cancel = rffi.llexternal("vmp_dyn_teardown", [lltype.Void],
+                                  rffi.INT, compilation_info=eci,
+                                  _nowrapper=True)
+
     return CInterface(locals())
 
 
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -161,16 +161,6 @@
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
 
 
-    def dyn_register_jit_page(self, addr, end_addr, loop, name=None):
-        if name is None:
-            cname = rffi.cast(rffi.CHARP, 0)
-        else:
-            cname = rffi.str2charp(name)
-        return self.cintf.vmp_dyn_register_jit_page(addr, end_addr, cname)
-
-    def dyn_cancel(self, ref):
-        self.cintf.vmp_dyn_cancel(ref)
-
     def _write_code_registration(self, uid, name):
         assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
             "the name must be 'class:func_name:func_line:filename' "
@@ -257,6 +247,26 @@
 
     return decorate
 
+def dyn_register_jit_page(token, addr, end_addr):
+    try:
+        c = _get_vmprof().cintf
+        cname = lltype.nullptr(rffi.CCHARP.TO)
+        ref = c.vmp_dyn_register_jit_page(addr, end_addr, cname)
+        token.rvmprof_register(ref)
+        return True
+    except cintf.VMProfPlatformUnsupported:
+        return False
+
+def dyn_cancel(token):
+    try:
+        c = _get_vmprof().cintf
+        for ref in token._rvmprof_references:
+            c.vmp_dyn_cancel(ref)
+        token._rvmprof_references = []
+        return True
+    except cintf.VMProfPlatformUnsupported:
+        return False
+
 @specialize.memo()
 def _was_registered(CodeClass):
     return hasattr(CodeClass, '_vmprof_unique_id')
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
@@ -9,8 +9,17 @@
 static int g_dyn_entry_count_max = 0;
 static unw_dyn_info_t ** g_dyn_entries = 0;
 
+RPY_EXTERN
 int vmp_dyn_teardown(void)
 {
+    int i;
+    for (i = 0; i < g_dyn_entry_count; i++) {
+        unw_dyn_info_t * u = g_dyn_entries[i];
+        if (u != NULL) {
+            free(u);
+            g_dyn_entries[i] = NULL;
+        }
+    }
     if (g_dyn_entries != NULL) {
         free(g_dyn_entries);
     }
@@ -30,7 +39,7 @@
         g_dyn_entry_count_max *= 2;
         g_dyn_entries = (unw_dyn_info_t**)realloc(g_dyn_entries, sizeof(unw_dyn_info_t*) * g_dyn_entry_count_max);
         memset(g_dyn_entries + g_dyn_entry_count, 0,
-               sizeof(unw_dyn_info_t*)*g_dyn_entry_count_max - g_dyn_entry_count);
+               sizeof(unw_dyn_info_t*)*(g_dyn_entry_count_max - g_dyn_entry_count));
     }
 }
 
@@ -72,6 +81,7 @@
     free(u);
 }
 
+RPY_EXTERN
 int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr,
                               const char * name)
 {
@@ -98,6 +108,7 @@
     return ref;
 }
 
+RPY_EXTERN
 int vmp_dyn_cancel(int ref) {
     unw_dyn_info_t * u;
 
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
--- a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
+++ b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
@@ -3,8 +3,10 @@
 #include <stdint.h>
 #include <libunwind.h>
 
-int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr,
+#include "rvmprof.h"
+
+RPY_EXTERN int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr,
                               const char * name);
-int vmp_dyn_cancel(int ref);
-int vmp_dyn_teardown(void);
+RPY_EXTERN int vmp_dyn_cancel(int ref);
+RPY_EXTERN int vmp_dyn_teardown(void);
 
diff --git a/rpython/rlib/rvmprof/test/test_dynamic.py b/rpython/rlib/rvmprof/test/test_dynamic.py
--- a/rpython/rlib/rvmprof/test/test_dynamic.py
+++ b/rpython/rlib/rvmprof/test/test_dynamic.py
@@ -16,11 +16,12 @@
         ffi.cdef("""
         int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr, const char * name);
         int vmp_dyn_cancel(int ref);
+        int vmp_dyn_teardown(void);
         """)
 
         with open(str(srcdir.join("shared/vmp_dynamic.c"))) as fd:
             ffi.set_source("rpython.rlib.rvmprof.test._test_dynamic", fd.read(),
-                    include_dirs=[str(srcdir.join('shared'))],
+                    include_dirs=[str(srcdir), str(srcdir.join('shared'))],
                     libraries=['unwind'])
 
         ffi.compile(verbose=True)
@@ -44,4 +45,28 @@
         lib.vmp_dyn_cancel(0)
         lib.vmp_dyn_cancel(1)
 
+    def test_register_dynamic_code_many(self):
+        lib = self.lib
+        ffi = self.ffi
 
+        refs = []
+        for i in range(5000):
+            ref = lib.vmp_dyn_register_jit_page(0x100*i, 0x200*i, ffi.NULL)
+            refs.append(ref)
+
+        for i in range(1000, 4000):
+            ref = refs[i]
+            lib.vmp_dyn_cancel(ref)
+
+        refs = refs[:1000] + refs[4000:]
+
+        for i in range(5000):
+            ref = lib.vmp_dyn_register_jit_page(0x100*i, 0x200*i, ffi.NULL)
+            refs.append(ref)
+
+        while refs:
+            ref = refs.pop()
+            lib.vmp_dyn_cancel(ref)
+
+        lib.vmp_dyn_teardown()
+

From pypy.commits at gmail.com  Sat Feb 11 11:37:50 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Sat, 11 Feb 2017 08:37:50 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: rename callsite method name,
 handle stack walking for jit frames
Message-ID: <589f3dde.829bdf0a.7973e.57f5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90051:ba06665d9168
Date: 2017-02-11 17:37 +0100
http://bitbucket.org/pypy/pypy/changeset/ba06665d9168/

Log:	rename callsite method name, handle stack walking for jit frames

diff --git a/rpython/jit/metainterp/memmgr.py b/rpython/jit/metainterp/memmgr.py
--- a/rpython/jit/metainterp/memmgr.py
+++ b/rpython/jit/metainterp/memmgr.py
@@ -71,7 +71,7 @@
         for looptoken in self.alive_loops.keys():
             if (0 <= looptoken.generation < max_generation or
                 looptoken.invalidated):
-                rvmprof.vmp_dyn_cancel(looptoken)
+                rvmprof.dyn_cancel(looptoken)
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -241,10 +241,15 @@
             // this is possible because compiler align to 8 bytes.
             //
 #ifdef PYPY_JIT_CODEMAP
-            if (top_most_frame->kind == VMPROF_JITTED_TAG) {
+            if (pip.format == UNW_INFO_FORMAT_DYNAMIC) {
+                if (top_most_frame->kind != VMPROF_JITTED_TAG) {
+                    // if this is encountered frequently something is wrong with
+                    // the stack building
+                    return 0;
+                }
                 intptr_t pc = ((intptr_t*)(top_most_frame->value - sizeof(intptr_t)))[0];
                 depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth);
-                frame = FRAME_STEP(frame);
+                top_most_frame = FRAME_STEP(top_most_frame);
             } else if (func_addr != 0x0) {
                 depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
             }

From pypy.commits at gmail.com  Sat Feb 11 12:09:54 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sat, 11 Feb 2017 09:09:54 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Add myself to the list.
Message-ID: <589f4562.85df1c0a.2f5b6.65df@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: extradoc
Changeset: r5779:e004bc384159
Date: 2017-02-11 18:08 +0100
http://bitbucket.org/pypy/extradoc/changeset/e004bc384159/

Log:	Add myself to the list.

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -12,6 +12,7 @@
 Armin Rigo                          private
 Richard Plangger     26.02/04.03    Ermina
 Remi Meier           (?)27.02/04.03 Ermina
+Manuel Jacob         25.02/04.03    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Sat Feb 11 12:09:52 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sat, 11 Feb 2017 09:09:52 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: I assume that they don't want to
 stay *that* long.
Message-ID: <589f4560.ad8ddf0a.e61b6.630e@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: extradoc
Changeset: r5778:733c6a0f438f
Date: 2017-02-11 18:07 +0100
http://bitbucket.org/pypy/extradoc/changeset/733c6a0f438f/

Log:	I assume that they don't want to stay *that* long.

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -10,8 +10,8 @@
     Name              Arrive/Depart     Accomodation 
 ==================== ============== =======================
 Armin Rigo                          private
-Richard Plangger     26.02/04.02    Ermina
-Remi Meier           (?)27.02/04.02 Ermina
+Richard Plangger     26.02/04.03    Ermina
+Remi Meier           (?)27.02/04.03 Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Sat Feb 11 12:22:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 09:22:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Try to expose in a new,
 pypy-only attribute '__defaults_count__' the
Message-ID: <589f4860.977d1c0a.74b95.0c6e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90052:c4cfe1d12253
Date: 2017-02-11 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/c4cfe1d12253/

Log:	Try to expose in a new, pypy-only attribute '__defaults_count__' the
	length of the internal '__defaults__', which might be hidden on some
	built-in functions because it would contain interp-level Nones.

diff --git a/lib-python/3/inspect.py b/lib-python/3/inspect.py
--- a/lib-python/3/inspect.py
+++ b/lib-python/3/inspect.py
@@ -2085,6 +2085,16 @@
     return _signature_fromstr(cls, func, s, skip_bound_arg)
 
 
+class _NoValue:
+    """Class of a marker object for PyPy only, used as the defaults for
+    built-in functions when there is really no Python object that could
+    be used."""
+    __slots__ = ()
+    def __repr__(self):
+        return '<no value>'
+_no_value = _NoValue()
+
+
 def _signature_from_function(cls, func):
     """Private helper: constructs Signature for the given python function."""
 
@@ -2113,7 +2123,10 @@
     if defaults:
         pos_default_count = len(defaults)
     else:
-        pos_default_count = 0
+        # PyPy extension, for built-in functions that take optional
+        # arguments but without any Python object to use as default.
+        pos_default_count = getattr(func, '__defaults_count__', 0)
+        defaults = [_no_value] * pos_default_count
 
     parameters = []
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -367,6 +367,9 @@
             return space.w_None
         return space.newtuple(values_w)
 
+    def fget_defaults_count(self, space):
+        return space.newint(len(self.defs_w))
+
     def fset_func_defaults(self, space, w_defaults):
         if space.is_w(w_defaults, space.w_None):
             self.defs_w = []
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -816,3 +816,5 @@
         w_g = space.wrap(app_g)
         w_defs = space.getattr(w_g, space.wrap("__defaults__"))
         assert space.is_w(w_defs, space.w_None)
+        w_count = space.getattr(w_g, space.wrap("__defaults_count__"))
+        assert space.unwrap(w_count) == 1
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -701,6 +701,7 @@
     __qualname__ = getset_func_qualname,
     __dict__ = getset_func_dict,
     __defaults__ = getset_func_defaults,
+    __defaults_count__ = GetSetProperty(Function.fget_defaults_count),
     __kwdefaults__ = getset_func_kwdefaults,
     __annotations__ = getset_func_annotations,
     __globals__ = interp_attrproperty_w('w_func_globals', cls=Function),

From pypy.commits at gmail.com  Sat Feb 11 12:26:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 09:26:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: document more c4cfe1d12253
Message-ID: <589f493e.8a6d1c0a.22c16.780b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90053:f1b0b8f3f631
Date: 2017-02-11 18:25 +0100
http://bitbucket.org/pypy/pypy/changeset/f1b0b8f3f631/

Log:	document more c4cfe1d12253

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -368,6 +368,12 @@
         return space.newtuple(values_w)
 
     def fget_defaults_count(self, space):
+        # PyPy extension.  This is mostly to make inspect.py happy about
+        # methods like dict.pop(), which take a default but is None.
+        # Previously, inspect.py would build a signature saying that all
+        # arguments are required, and then unittest.mock would complain
+        # if we call it with less arguments.  Now inspect.py looks up
+        # '__defaults_count__' too.
         return space.newint(len(self.defs_w))
 
     def fset_func_defaults(self, space, w_defaults):

From pypy.commits at gmail.com  Sat Feb 11 12:32:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 09:32:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Skip the final check in this test,
 with comment
Message-ID: <589f4ab7.d3471c0a.91f2.6bb3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90054:48030bda1edd
Date: 2017-02-11 18:32 +0100
http://bitbucket.org/pypy/pypy/changeset/48030bda1edd/

Log:	Skip the final check in this test, with comment

diff --git a/lib-python/3/test/test_signal.py b/lib-python/3/test/test_signal.py
--- a/lib-python/3/test/test_signal.py
+++ b/lib-python/3/test/test_signal.py
@@ -580,9 +580,11 @@
 
         err = err.getvalue()
         if ('Exception ignored when trying to {action} to the signal wakeup fd'
-            not in err):
+            not in err) and {cpython_only}:
             raise AssertionError(err)
-        """.format(action=action)
+        """.format(action=action, cpython_only=support.check_impl_detail())
+        # note that PyPy produces the same error message, but sent to
+        # the real stderr instead of to sys.stderr.
         assert_python_ok('-c', code)
 
 

From pypy.commits at gmail.com  Sat Feb 11 12:38:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Feb 2017 09:38:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix the expected exception messages
Message-ID: <589f4c20.125e1c0a.f9b46.169f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90055:c7f45f48949e
Date: 2017-02-11 18:38 +0100
http://bitbucket.org/pypy/pypy/changeset/c7f45f48949e/

Log:	fix the expected exception messages

diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -1064,8 +1064,9 @@
                         self.assertIn("<exception str() failed>", report)
                     else:
                         # pypy: this is what lib-python's traceback.py gives
-                        self.assertIn("<unprintable BrokenExceptionDel object>",
-                                      report)
+                        self.assertRegex(report,
+                            ".*BrokenStrException: <unprintable"
+                            " BrokenStrException object>\n")
                 else:
                     self.assertIn("ValueError", report)
                     self.assertIn("del is broken", report)
@@ -1118,7 +1119,8 @@
         self.assertEqual(exc.name, 'somename')
         self.assertEqual(exc.path, 'somepath')
 
-        msg = "'invalid' is an invalid keyword argument for this function"
+        msg = ("'invalid' is an invalid keyword argument for this function"
+               "|ImportError does not take keyword arguments")  # PyPy message
         with self.assertRaisesRegex(TypeError, msg):
             ImportError('test', invalid='keyword')
 
@@ -1131,7 +1133,8 @@
         with self.assertRaisesRegex(TypeError, msg):
             ImportError(invalid='keyword')
 
-        msg = "'invalid|another' is an invalid keyword argument for this function"
+        msg = ("'invalid|another' is an invalid keyword argument for this function"
+               "|ImportError does not take keyword arguments")  # PyPy message
         with self.assertRaisesRegex(TypeError, msg):
             ImportError('test', invalid='keyword', another=True)
 

From pypy.commits at gmail.com  Sat Feb 11 13:00:48 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 11 Feb 2017 10:00:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Update PyLong_AsLong docstring
Message-ID: <589f5150.f0a6df0a.b1e14.703b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90056:43a08e490b5b
Date: 2017-02-11 17:59 +0000
http://bitbucket.org/pypy/pypy/changeset/43a08e490b5b/

Log:	Update PyLong_AsLong docstring

diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -70,9 +70,9 @@
 @cpython_api([PyObject], lltype.Signed, error=-1)
 def PyLong_AsLong(space, w_long):
     """
-    Return a C long representation of the contents of pylong.  If
-    pylong is greater than LONG_MAX, an OverflowError is raised
-    and -1 will be returned."""
+    Get a C long int from an int object or any object that has an __int__
+    method.  Return -1 and set an error if overflow occurs.
+    """
     return space.int_w(space.int(w_long))
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)

From pypy.commits at gmail.com  Sat Feb 11 14:17:34 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 11 Feb 2017 11:17:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Since 3.3,
 0-D memoryviews return an empty tuple for .shape and .strides (hard
 to test at app-level)
Message-ID: <589f634e.8ba4df0a.ebc3b.80ec@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90057:e06ef56cac01
Date: 2017-02-11 19:16 +0000
http://bitbucket.org/pypy/pypy/changeset/e06ef56cac01/

Log:	Since 3.3, 0-D memoryviews return an empty tuple for .shape and
	.strides (hard to test at app-level)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -323,11 +323,11 @@
         self.w_obj = w_obj # kept alive
         self.pyobj = make_ref(space, w_obj)
         self.format = format
-        if not shape:
+        if shape is None:
             self.shape = [size]
         else:
             self.shape = shape
-        if not strides:
+        if strides is None:
             self.strides = [1]
         else:
             self.strides = strides
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -428,14 +428,10 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        if self.getndim() == 0:
-            return space.w_None
         return space.newtuple([space.wrap(x) for x in self.getshape()])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        if self.getndim() == 0:
-            return space.w_None
         return space.newtuple([space.wrap(x) for x in self.getstrides()])
 
     def w_get_suboffsets(self, space):

From pypy.commits at gmail.com  Sat Feb 11 14:42:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 11 Feb 2017 11:42:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test and partial fix for 0d memoryview
Message-ID: <589f6922.cc881c0a.512e1.35cb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90058:de015dfe7aff
Date: 2017-02-11 19:41 +0000
http://bitbucket.org/pypy/pypy/changeset/de015dfe7aff/

Log:	Test and partial fix for 0d memoryview

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -565,7 +565,7 @@
 
     def _init_flags(self):
         buf = self.buf
-        ndim = buf.getndim()
+        ndim = self.getndim()
         flags = 0
         if ndim == 0:
             flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -27,6 +27,14 @@
         assert len(w) == 2
         exc = raises(TypeError, "memoryview('foobar')")
 
+    def test_0d(self):
+        v = memoryview(b'x').cast('B', ())
+        assert len(v) == 1
+        assert v.shape == ()
+        assert v.strides == ()
+        #assert v.tobytes() == b'x'
+        #assert v[()] == b'x'[0]
+
     def test_rw(self):
         data = bytearray(b'abcefg')
         v = memoryview(data)

From pypy.commits at gmail.com  Sat Feb 11 14:56:24 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 11 Feb 2017 11:56:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix .tobytes() on 0d memoryview
Message-ID: <589f6c68.b086df0a.ceffa.8c25@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90059:9e067f394eee
Date: 2017-02-11 19:55 +0000
http://bitbucket.org/pypy/pypy/changeset/9e067f394eee/

Log:	Fix .tobytes() on 0d memoryview

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -117,6 +117,9 @@
         return ''.join(self.copy_buffer())
 
     def copy_buffer(self):
+        if self.getndim() == 0:
+            itemsize = self.getitemsize()
+            return [self.buf.getslice(0, itemsize, 1, itemsize)]
         data = []
         self._copy_rec(0, data, 0)
         return data
@@ -126,12 +129,12 @@
         shape = shapes[idim]
         strides = self.getstrides()
 
-        if self.getndim()-1 == idim:
-            self._copy_base(data,off)
+        if self.getndim() - 1 == idim:
+            self._copy_base(data, off)
             return
 
         for i in range(shape):
-            self._copy_rec(idim+1,data,off)
+            self._copy_rec(idim + 1, data, off)
             off += strides[idim]
 
     def _copy_base(self, data, off):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -32,7 +32,7 @@
         assert len(v) == 1
         assert v.shape == ()
         assert v.strides == ()
-        #assert v.tobytes() == b'x'
+        assert v.tobytes() == b'x'
         #assert v[()] == b'x'[0]
 
     def test_rw(self):

From pypy.commits at gmail.com  Sat Feb 11 15:11:16 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 11 Feb 2017 12:11:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test 0d view on C-created buffer
Message-ID: <589f6fe4.cce61c0a.3c70a.a4f7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90060:9f7edc16f16c
Date: 2017-02-11 20:10 +0000
http://bitbucket.org/pypy/pypy/changeset/9f7edc16f16c/

Log:	Test 0d view on C-created buffer

diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -69,6 +69,14 @@
                                 b'\x03\0\0\0'
                                 b'\x04\0\0\0')
 
+    def test_0d_view(self):
+        module = self.import_module(name='array')
+        arr = module.array('B', b'\0\0\0\x01')
+        buf = memoryview(arr).cast('i', shape=())
+        assert bytes(buf) == b'\0\0\0\x01'
+        assert buf.shape == ()
+        assert buf.strides == ()
+
     def test_binop_mul_impl(self):
         # check that rmul is called
         module = self.import_module(name='array')

From pypy.commits at gmail.com  Sun Feb 12 03:24:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 00:24:59 -0800 (PST)
Subject: [pypy-commit] pypy default: Move OrderedDict.popitem() to
 interp-level as an attempt to increase
Message-ID: <58a01bdb.ce9adf0a.287e7.1840@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90061:63aaa4e32c86
Date: 2017-02-12 09:24 +0100
http://bitbucket.org/pypy/pypy/changeset/63aaa4e32c86/

Log:	Move OrderedDict.popitem() to interp-level as an attempt to increase
	its multi-threaded resistence

diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -33,6 +33,10 @@
     from __pypy__ import reversed_dict as _reversed_dict
 except ImportError:
     _reversed_dict = None     # don't have ordered dicts
+try:
+    from __pypy__ import dict_popitem_first as _dict_popitem_first
+except ImportError:
+    _dict_popitem_first = None
 
 try:
     from thread import get_ident as _get_ident
@@ -44,6 +48,17 @@
 ### OrderedDict
 ################################################################################
 
+if _dict_popitem_first is None:
+    def _dict_popitem_first(self):
+        it = dict.iteritems(self)
+        try:
+            k, v = it.next()
+        except StopIteration:
+            raise KeyError('dictionary is empty')
+        dict.__delitem__(self, k)
+        return (k, v)
+
+
 class OrderedDict(dict):
     '''Dictionary that remembers insertion order.
 
@@ -68,12 +83,7 @@
         if last:
             return dict.popitem(self)
         else:
-            it = dict.__iter__(self)
-            try:
-                k = it.next()
-            except StopIteration:
-                raise KeyError('dictionary is empty')
-            return (k, self.pop(k))
+            return _dict_popitem_first(self)
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -79,6 +79,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
+        'dict_popitem_first'        : 'interp_dict.dict_popitem_first',
         'delitem_if_value_is'       : 'interp_dict.delitem_if_value_is',
         'move_to_end'               : 'interp_dict.move_to_end',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -45,6 +45,14 @@
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_reversed_dict(space)
 
+def dict_popitem_first(space, w_obj):
+    """Interp-level implementation of OrderedDict.popitem(last=False).
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_popitem_first(space)
+
 def delitem_if_value_is(space, w_obj, w_key, w_value):
     """Atomic equivalent to: 'if dict.get(key) is value: del dict[key]'.
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -338,6 +338,15 @@
                     for i in range(len(keys_w)):
                         self.setitem(keys_w[i], values_w[i])
 
+    def nondescr_popitem_first(self, space):
+        """Not exposed directly to app-level, but via __pypy__.popitem_first().
+        """
+        w_key, w_value = self.iteritems().next_item()
+        if w_key is None:
+            raise oefmt(space.w_KeyError, "popitem(): dictionary is empty")
+        self.delitem(w_key)
+        return space.newtuple([w_key, w_value])
+
     def descr_clear(self, space):
         """D.clear() -> None.  Remove all items from D."""
         self.clear()
@@ -604,8 +613,7 @@
 
     has_iterreversed = False
     has_move_to_end = False
-    # no 'getiterreversed' and no 'move_to_end': no default
-    # implementation available
+    # ^^^ no default implementation available for these methods
 
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         iteritems = self.iteritems(w_dict)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -267,10 +267,22 @@
         d = {1: 2, 3: 4, 5: 6}
         it = __pypy__.reversed_dict(d)
         key = it.next()
-        assert key in [1, 3, 5]
+        assert key in [1, 3, 5]   # on CPython, dicts are not ordered
         del d[key]
         raises(RuntimeError, it.next)
 
+    def test_dict_popitem_first(self):
+        import __pypy__
+        d = {"a": 5}
+        assert __pypy__.dict_popitem_first(d) == ("a", 5)
+        raises(KeyError, __pypy__.dict_popitem_first, d)
+
+        def kwdict(**k):
+            return k
+        d = kwdict(a=55)
+        assert __pypy__.dict_popitem_first(d) == ("a", 55)
+        raises(KeyError, __pypy__.dict_popitem_first, d)
+
     def test_delitem_if_value_is(self):
         import __pypy__
         class X:

From pypy.commits at gmail.com  Sun Feb 12 03:46:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 00:46:44 -0800 (PST)
Subject: [pypy-commit] pypy default: dict.pop() on a key with a strange
 __eq__ could crash pypy
Message-ID: <58a020f4.2fa0df0a.942d3.1757@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90062:943aa7094f15
Date: 2017-02-12 09:46 +0100
http://bitbucket.org/pypy/pypy/changeset/943aa7094f15/

Log:	dict.pop() on a key with a strange __eq__ could crash pypy

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -237,6 +237,14 @@
         except KeyError:
             space.raise_key_error(w_key)
 
+    def internal_delitem(self, w_key):
+        try:
+            self.delitem(w_key)
+        except KeyError:
+            raise oefmt(self.space.w_RuntimeError,
+                        "an internal 'del' on the dictionary failed to find "
+                        "the key")
+
     def descr_reversed(self, space):
         raise oefmt(space.w_TypeError,
                     "argument to reversed() must be a sequence")
@@ -369,7 +377,7 @@
             else:
                 space.raise_key_error(w_key)
         else:
-            self.delitem(w_key)
+            self.internal_delitem(w_key)
             return w_item
 
     def descr_popitem(self, space):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -735,6 +735,31 @@
         setattr(a, s, 123)
         assert holder.seen is s
 
+    def test_internal_delitem(self):
+        class K:
+            def __hash__(self):
+                return 42
+            def __eq__(self, other):
+                if is_equal[0]:
+                    is_equal[0] -= 1
+                    return True
+                return False
+        is_equal = [0]
+        k1 = K()
+        k2 = K()
+        d = {k1: 1, k2: 2}
+        k3 = K()
+        is_equal = [1]
+        try:
+            x = d.pop(k3)
+        except RuntimeError:
+            # This used to give a Fatal RPython error: KeyError.
+            # Now at least it should raise an app-level RuntimeError,
+            # or just work.
+            assert len(d) == 2
+        else:
+            assert (x == 1 or x == 2) and len(d) == 1
+
 
 class AppTestDictViews:
     def test_dictview(self):

From pypy.commits at gmail.com  Sun Feb 12 03:48:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 00:48:19 -0800 (PST)
Subject: [pypy-commit] pypy default: more places (more messy to test)
Message-ID: <58a02153.aa8adf0a.aee19.135f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90063:3f205d62f5d4
Date: 2017-02-12 09:47 +0100
http://bitbucket.org/pypy/pypy/changeset/3f205d62f5d4/

Log:	more places (more messy to test)

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -327,7 +327,7 @@
             if w_value is None:
                 space.raise_key_error(w_key)
             else:
-                self.delitem(w_key)
+                self.internal_delitem(w_key)
                 if last_flag:
                     self.setitem(w_key, w_value)
                 else:
@@ -352,7 +352,7 @@
         w_key, w_value = self.iteritems().next_item()
         if w_key is None:
             raise oefmt(space.w_KeyError, "popitem(): dictionary is empty")
-        self.delitem(w_key)
+        self.internal_delitem(w_key)
         return space.newtuple([w_key, w_value])
 
     def descr_clear(self, space):

From pypy.commits at gmail.com  Sun Feb 12 04:01:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 01:01:04 -0800 (PST)
Subject: [pypy-commit] pypy default: inline missing_method(),
 which avoids re-checking some values for None
Message-ID: <58a02450.c3a4df0a.6ec44.1ab6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90064:ae54260458e4
Date: 2017-02-12 09:57 +0100
http://bitbucket.org/pypy/pypy/changeset/ae54260458e4/

Log:	inline missing_method(), which avoids re-checking some values for
	None

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -102,13 +102,6 @@
             result[key] = val
         return result
 
-    def missing_method(w_dict, space, w_key):
-        if not space.is_w(space.type(w_dict), space.w_dict):
-            w_missing = space.lookup(w_dict, '__missing__')
-            if w_missing is not None:
-                return space.get_and_call_function(w_missing, w_dict, w_key)
-        return None
-
     def initialize_content(self, list_pairs_w):
         for w_k, w_v in list_pairs_w:
             self.setitem(w_k, w_v)
@@ -222,9 +215,11 @@
         if w_value is not None:
             return w_value
 
-        w_missing_item = self.missing_method(space, w_key)
-        if w_missing_item is not None:
-            return w_missing_item
+        # if there is a __missing__ method, call it
+        if not space.is_w(space.type(self), space.w_dict):
+            w_missing = space.lookup(self, '__missing__')
+            if w_missing is not None:
+                return space.get_and_call_function(w_missing, self, w_key)
 
         space.raise_key_error(w_key)
 

From pypy.commits at gmail.com  Sun Feb 12 04:01:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 01:01:06 -0800 (PST)
Subject: [pypy-commit] pypy default: In the common strategies,
 implement a version of pop() doing only one dict lookup
Message-ID: <58a02452.53871c0a.9f38c.2ce3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90065:6b030e17ad91
Date: 2017-02-12 09:58 +0100
http://bitbucket.org/pypy/pypy/changeset/6b030e17ad91/

Log:	In the common strategies, implement a version of pop() doing only
	one dict lookup

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -365,6 +365,13 @@
         corresponding value\nIf key is not found, d is returned if given,
         otherwise KeyError is raised
         """
+        strategy = self.get_strategy()
+        if strategy.has_pop:
+            try:
+                return strategy.pop(self, w_key, w_default)
+            except KeyError:
+                raise space.raise_key_error(w_key)
+        # fall-back
         w_item = self.getitem(w_key)
         if w_item is None:
             if w_default is not None:
@@ -616,6 +623,7 @@
 
     has_iterreversed = False
     has_move_to_end = False
+    has_pop = False
     # ^^^ no default implementation available for these methods
 
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
@@ -890,6 +898,9 @@
     if hasattr(dictimpl, 'move_to_end'):
         dictimpl.has_move_to_end = True
 
+    if hasattr(dictimpl, 'pop'):
+        dictimpl.has_pop = True
+
     @jit.look_inside_iff(lambda self, w_dict, w_updatedict:
                          w_dict_unrolling_heuristic(w_dict))
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
@@ -1028,6 +1039,21 @@
         key, value = self.unerase(w_dict.dstorage).popitem()
         return (self.wrap(key), value)
 
+    def pop(self, w_dict, w_key, w_default):
+        space = self.space
+        if self.is_correct_type(w_key):
+            key = self.unwrap(w_key)
+            d = self.unerase(w_dict.dstorage)
+            if w_default is None:
+                return d.pop(key)
+            else:
+                return d.pop(key, w_default)
+        elif self._never_equal_to(space.type(w_key)):
+            raise KeyError
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.get_strategy().pop(w_dict, w_key, w_default)
+
     def clear(self, w_dict):
         self.unerase(w_dict.dstorage).clear()
 
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -663,6 +663,18 @@
             else:
                 assert False, 'Expected KeyError'
 
+    def test_pop_switching_strategy(self):
+        class Foo:
+            def __hash__(self):
+                return hash("a")
+            def __eq__(self, other):
+                return other == "a"
+        d = {"a": 42}
+        x = d.pop(Foo())
+        assert x == 42 and len(d) == 0
+        d = {"b": 43}
+        raises(KeyError, d.pop, Foo())
+
     def test_no_len_on_dict_iter(self):
         iterable = {1: 2, 3: 4}
         raises(TypeError, len, iter(iterable))

From pypy.commits at gmail.com  Sun Feb 12 04:40:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 01:40:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Direct access to the interp-level
 fsencode/fsdecode, for tests
Message-ID: <58a02d97.484a1c0a.a54ba.336a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90066:e697cc164d5a
Date: 2017-02-12 10:15 +0100
http://bitbucket.org/pypy/pypy/changeset/e697cc164d5a/

Log:	Direct access to the interp-level fsencode/fsdecode, for tests

diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
--- a/pypy/interpreter/test/test_fsencode.py
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -87,3 +87,17 @@
         # ValueError directly, or return a wrapped bytes with the 0
         # embedded---and then space.fsencode_w() should raise ValueError.
         space.raises_w(space.w_ValueError, space.fsencode_w, w_u)
+
+    def test_interface_from___pypy__(self):
+        space = self.space
+        strs = [u"/home/bar/baz", u"c:\\"]
+        if self.special_char:
+            strs.append(self.special_char)
+        for st in strs:
+            w_st = space.newunicode(st)
+            w_enc = space.fsencode(w_st)
+            space.appexec([w_st, w_enc], """(u, s):
+                import __pypy__
+                assert __pypy__.fsencode(u) == s
+                assert __pypy__.fsdecode(s) == u
+            """)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -91,6 +91,8 @@
         'normalize_exc'             : 'interp_magic.normalize_exc',
         'StdErrPrinter'             : 'interp_stderrprinter.W_StdErrPrinter',
         'stack_almost_full'         : 'interp_magic.stack_almost_full',
+        'fsencode'                  : 'interp_magic.fsencode',
+        'fsdecode'                  : 'interp_magic.fsdecode',
     }
 
     submodules = {
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -180,3 +180,11 @@
 def stack_almost_full(space):
     """Return True if the stack is more than 15/16th full."""
     return space.wrap(rstack.stack_almost_full())
+
+def fsencode(space, w_obj):
+    """Direct access to the interp-level fsencode()"""
+    return space.fsencode(w_obj)
+
+def fsdecode(space, w_obj):
+    """Direct access to the interp-level fsdecode()"""
+    return space.fsdecode(w_obj)

From pypy.commits at gmail.com  Sun Feb 12 04:40:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 01:40:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <58a02d98.e780df0a.771d3.1f02@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90067:794944a8179b
Date: 2017-02-12 10:24 +0100
http://bitbucket.org/pypy/pypy/changeset/794944a8179b/

Log:	fix

diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -54,7 +54,7 @@
         with scoped_unicode2rawwcharp(u) as ubuf:
             sbuf = pypy_wchar2char(ubuf, errorposp)
         try:
-            if sbuf is None:
+            if not sbuf:
                 errorpos = rffi.cast(lltype.Signed, errorposp[0])
                 if errorpos == -1:
                     raise MemoryError
@@ -80,7 +80,7 @@
         with rffi.scoped_str2charp(s) as sbuf:
             ubuf = pypy_char2wchar(sbuf, sizep)
         try:
-            if ubuf is None:
+            if not ubuf:
                 errmsg = _errmsg(u"pypy_char2wchar")
                 errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
             size = rffi.cast(lltype.Signed, sizep[0])
@@ -90,9 +90,8 @@
 
 
 def _errmsg(what):
-    from rpython.rlib import rposix
-    errmsg = _strerror(rposix.get_errno())
-    return u"%s failed" % what if errmsg is None else errmsg
+    # I *think* that the functions in locale_codec.c don't set errno
+    return u"%s failed" % what
 
 
 class scoped_unicode2rawwcharp:

From pypy.commits at gmail.com  Sun Feb 12 04:40:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 01:40:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <58a02d9c.0f96df0a.648cf.2523@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90069:ffcc62f39db7
Date: 2017-02-12 10:40 +0100
http://bitbucket.org/pypy/pypy/changeset/ffcc62f39db7/

Log:	merge heads

diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -70,9 +70,9 @@
 @cpython_api([PyObject], lltype.Signed, error=-1)
 def PyLong_AsLong(space, w_long):
     """
-    Return a C long representation of the contents of pylong.  If
-    pylong is greater than LONG_MAX, an OverflowError is raised
-    and -1 will be returned."""
+    Get a C long int from an int object or any object that has an __int__
+    method.  Return -1 and set an error if overflow occurs.
+    """
     return space.int_w(space.int(w_long))
 
 @cpython_api([PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -323,11 +323,11 @@
         self.w_obj = w_obj # kept alive
         self.pyobj = make_ref(space, w_obj)
         self.format = format
-        if not shape:
+        if shape is None:
             self.shape = [size]
         else:
             self.shape = shape
-        if not strides:
+        if strides is None:
             self.strides = [1]
         else:
             self.strides = strides
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -69,6 +69,14 @@
                                 b'\x03\0\0\0'
                                 b'\x04\0\0\0')
 
+    def test_0d_view(self):
+        module = self.import_module(name='array')
+        arr = module.array('B', b'\0\0\0\x01')
+        buf = memoryview(arr).cast('i', shape=())
+        assert bytes(buf) == b'\0\0\0\x01'
+        assert buf.shape == ()
+        assert buf.strides == ()
+
     def test_binop_mul_impl(self):
         # check that rmul is called
         module = self.import_module(name='array')
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -117,6 +117,9 @@
         return ''.join(self.copy_buffer())
 
     def copy_buffer(self):
+        if self.getndim() == 0:
+            itemsize = self.getitemsize()
+            return [self.buf.getslice(0, itemsize, 1, itemsize)]
         data = []
         self._copy_rec(0, data, 0)
         return data
@@ -126,12 +129,12 @@
         shape = shapes[idim]
         strides = self.getstrides()
 
-        if self.getndim()-1 == idim:
-            self._copy_base(data,off)
+        if self.getndim() - 1 == idim:
+            self._copy_base(data, off)
             return
 
         for i in range(shape):
-            self._copy_rec(idim+1,data,off)
+            self._copy_rec(idim + 1, data, off)
             off += strides[idim]
 
     def _copy_base(self, data, off):
@@ -428,14 +431,10 @@
 
     def w_get_shape(self, space):
         self._check_released(space)
-        if self.getndim() == 0:
-            return space.w_None
         return space.newtuple([space.wrap(x) for x in self.getshape()])
 
     def w_get_strides(self, space):
         self._check_released(space)
-        if self.getndim() == 0:
-            return space.w_None
         return space.newtuple([space.wrap(x) for x in self.getstrides()])
 
     def w_get_suboffsets(self, space):
@@ -569,7 +568,7 @@
 
     def _init_flags(self):
         buf = self.buf
-        ndim = buf.getndim()
+        ndim = self.getndim()
         flags = 0
         if ndim == 0:
             flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -27,6 +27,14 @@
         assert len(w) == 2
         exc = raises(TypeError, "memoryview('foobar')")
 
+    def test_0d(self):
+        v = memoryview(b'x').cast('B', ())
+        assert len(v) == 1
+        assert v.shape == ()
+        assert v.strides == ()
+        assert v.tobytes() == b'x'
+        #assert v[()] == b'x'[0]
+
     def test_rw(self):
         data = bytearray(b'abcefg')
         v = memoryview(data)

From pypy.commits at gmail.com  Sun Feb 12 04:40:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 01:40:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: unicode fixes
Message-ID: <58a02d9a.0fb81c0a.ee3e2.3317@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90068:ec83c91917b2
Date: 2017-02-12 10:39 +0100
http://bitbucket.org/pypy/pypy/changeset/ec83c91917b2/

Log:	unicode fixes

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -140,7 +140,7 @@
     # setup filename
     try:
         w_filename = space.getitem(w_globals, space.wrap("__file__"))
-        filename = space.str_w(w_filename)
+        filename = space.fsencode_w(w_filename)
     except OperationError as e:
         if space.str_w(w_module) == '__main__':
             w_argv = space.sys.getdictvalue(space, 'argv')
@@ -157,7 +157,7 @@
         lc_filename = filename.lower()
         if lc_filename.endswith(".pyc"):
             # strip last character
-            w_filename = space.wrap(filename[:-1])
+            w_filename = space.fsdecode(space.newbytes(filename[:-1]))
 
     return (w_filename, lineno, w_module, w_registry)
 
@@ -242,9 +242,10 @@
     w_stderr = space.sys.get("stderr")
 
     # Print "filename:lineno: category: text\n"
-    message = "%s:%d: %s: %s\n" % (space.str_w(w_filename), lineno,
-                                   space.str_w(w_name), space.str_w(w_text))
-    space.call_method(w_stderr, "write", space.wrap(message))
+    message = u"%s:%d: %s: %s\n" % (space.unicode_w(w_filename), lineno,
+                                    space.unicode_w(w_name),
+                                    space.unicode_w(w_text))
+    space.call_method(w_stderr, "write", space.newunicode(message))
 
     # Print "  source_line\n"
     if not w_sourceline:
@@ -261,17 +262,17 @@
 
     if not w_sourceline:
         return
-    line = space.str_w(w_sourceline)
+    line = space.unicode_w(w_sourceline)
     if not line:
         return
 
-    message = "\n"
+    message = u"\n"
     for i in range(len(line)):
         c = line[i]
-        if c not in ' \t\014':
-            message = "  %s\n" % (line[i:],)
+        if c not in u' \t\014':
+            message = u"  %s\n" % (line[i:],)
             break
-    space.call_method(w_stderr, "write", space.wrap(message))
+    space.call_method(w_stderr, "write", space.newunicode(message))
 
 def do_warn(space, w_message, w_category, stacklevel):
     context_w = setup_context(space, stacklevel)
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -87,3 +87,12 @@
         class Foo:
             pass
         raises(TypeError, _warnings.warn, "text", Foo)
+
+    def test_surrogate_in_filename(self):
+        import _warnings, __pypy__
+        for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"):
+            try:
+                __pypy__.fsencode(filename)
+            except UnicodeEncodeError:
+                continue
+            _warnings.warn_explicit("text", UserWarning, filename, 1)

From pypy.commits at gmail.com  Sun Feb 12 12:09:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 09:09:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: f-strings should never show up as
 docstrings (cpython 3.6.1 bugfix)
Message-ID: <58a096b6.c80e1c0a.f9360.9675@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90070:012eb9df87bd
Date: 2017-02-12 18:08 +0100
http://bitbucket.org/pypy/pypy/changeset/012eb9df87bd/

Log:	f-strings should never show up as docstrings (cpython 3.6.1 bugfix)

diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
--- a/lib-python/3/test/test_fstring.py
+++ b/lib-python/3/test/test_fstring.py
@@ -1,4 +1,5 @@
-# This test file is from CPython 3.6.0
+# This test file is from CPython 3.6 rev. 30341d5c1423, with a few
+# changes
 
 import ast
 import types
@@ -72,18 +73,18 @@
         # Make sure x was called.
         self.assertTrue(x.called)
 
+    def test_docstring(self):
+        def f():
+            f'''Not a docstring'''
+        self.assertIsNone(f.__doc__)
+        def g():
+            '''Not a docstring''' \
+            f''
+        self.assertIsNone(g.__doc__)
+
     def test_literal_eval(self):
-        # With no expressions, an f-string is okay.
-        self.assertEqual(ast.literal_eval("f'x'"), 'x')
-        self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy')
-
-        # But this should raise an error.
         with self.assertRaisesRegex(ValueError, 'malformed node or string'):
-            ast.literal_eval("f'x{3}'")
-
-        # As should this, which uses a different ast node
-        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
-            ast.literal_eval("f'{3}'")
+            ast.literal_eval("f'x'")
 
     def test_ast_compile_time_concat(self):
         x = ['']
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1506,7 +1506,8 @@
         self.update_position(joinedstr.lineno)
         for node in joinedstr.values:
             node.walkabout(self)
-        self.emit_op_arg(ops.BUILD_STRING, len(joinedstr.values))
+        if len(joinedstr.values) != 1:
+            self.emit_op_arg(ops.BUILD_STRING, len(joinedstr.values))
 
     def visit_FormattedValue(self, fmt):
         fmt.value.walkabout(self)
diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -325,25 +325,23 @@
 
 
 def f_string_to_ast_node(astbuilder, joined_pieces, atom_node):
-    # remove empty Strs
+    # Remove empty Strs, but always return an ast.JoinedStr object.
+    # In this way it cannot be grabbed later for being used as a
+    # docstring.  In codegen.py we still special-case length-1 lists
+    # and avoid calling "BUILD_STRING 1" in this case.
     space = astbuilder.space
     values = [node for node in joined_pieces
                    if not isinstance(node, ast.Str)
                       or space.is_true(node.s)]
-    if len(values) > 1:
-        return ast.JoinedStr(values, atom_node.get_lineno(),
-                                     atom_node.get_column())
-    elif len(values) == 1:
-        return values[0]
-    else:
-        assert len(joined_pieces) > 0    # they are all empty strings
-        return joined_pieces[0]
+    return ast.JoinedStr(values, atom_node.get_lineno(),
+                                 atom_node.get_column())
 
 
 def string_parse_literal(astbuilder, atom_node):
     space = astbuilder.space
     encoding = astbuilder.compile_info.encoding
     joined_pieces = []
+    fmode = False
     try:
         for i in range(atom_node.num_children()):
             w_next = parsestring.parsestr(
@@ -353,6 +351,7 @@
                                     atom_node)
             else:
                 parse_f_string(astbuilder, joined_pieces, w_next, atom_node)
+                fmode = True
 
     except error.OperationError as e:
         if e.match(space, space.w_UnicodeError):
@@ -366,7 +365,7 @@
         errmsg = space.str_w(space.str(e.get_w_value(space)))
         raise astbuilder.error('(%s) %s' % (kind, errmsg), atom_node)
 
-    if len(joined_pieces) == 1:   # <= the common path
+    if not fmode and len(joined_pieces) == 1:   # <= the common path
         return joined_pieces[0]   # ast.Str, Bytes or FormattedValue
 
     # with more than one piece, it is a combination of Str and
@@ -376,4 +375,5 @@
         if isinstance(node, ast.Bytes):
             astbuilder.error("cannot mix bytes and nonbytes literals",
                              atom_node)
+    assert fmode
     return f_string_to_ast_node(astbuilder, joined_pieces, atom_node)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -417,6 +417,7 @@
                 foo = Foo()
                 exec("'moduledoc'", foo.__dict__)
              ''',                            "moduledoc"),
+            ('''def foo(): f"abc"''',        None),
             ]:
             yield self.simple_test, source, "foo.__doc__", expected
 

From pypy.commits at gmail.com  Sun Feb 12 12:31:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 09:31:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix time.__hash__
Message-ID: <58a09c0c.04abdf0a.c4902.8cd6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90071:af772545b69c
Date: 2017-02-12 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/af772545b69c/

Log:	Fix time.__hash__

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -1151,11 +1151,13 @@
             # tzoff, that is.  If we have tzoff then CPython's hashes
             # are again deterministic.  I have no clue why.  We'll go
             # for now for also being nondeterministic in this case.
-            temp1 = '%d@%d@%d@%d@%s' % (self._hour,
-                                        self._minute,
-                                        self._second,
-                                        self._microsecond,
-                                        self.utcoffset())
+            myhhmm = self._hour * 60 + self._minute
+            myoffset = self.utcoffset()
+            if myoffset is not None:
+                myhhmm -= myoffset // timedelta(minutes=1)
+            temp1 = '%d@%d@%d' % (myhhmm,
+                                  self._second,
+                                  self._microsecond)
             self._hashcode = hash(temp1)
         return self._hashcode
 

From pypy.commits at gmail.com  Sun Feb 12 12:31:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 09:31:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: datetime.__hash__() was still
 deterministic if tzoff!=None.
Message-ID: <58a09c0e.c6d11c0a.ac510.9f1c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90072:7e28f9b94af3
Date: 2017-02-12 18:30 +0100
http://bitbucket.org/pypy/pypy/changeset/7e28f9b94af3/

Log:	datetime.__hash__() was still deterministic if tzoff!=None.

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -1766,9 +1766,16 @@
             if tzoff is None:
                 self._hashcode = hash(self._getstate()[0])
             else:
+                # PyPy: uses an algo that relies on the hash of strings,
+                # giving a nondeterministic result.  CPython doesn't do
+                # that if there is a tzoff (but does if there is no
+                # tzoff).
                 days = _ymd2ord(self.year, self.month, self.day)
                 seconds = self.hour * 3600 + self.minute * 60 + self.second
-                self._hashcode = hash(timedelta(days, seconds, self.microsecond) - tzoff)
+                delta = timedelta(days, seconds, self.microsecond) - tzoff
+                temp1 = '%d&%d&%d' % (delta.days, delta.seconds,
+                                      delta.microseconds)
+                self._hashcode = hash(temp1)
         return self._hashcode
 
     # Pickle support.

From pypy.commits at gmail.com  Sun Feb 12 16:56:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Feb 2017 13:56:22 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58a0da06.8794df0a.c66a3.ca5c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r855:92030bfcabb9
Date: 2017-02-12 22:56 +0100
http://bitbucket.org/pypy/pypy.org/changeset/92030bfcabb9/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66591 of $105000 (63.4%)
+   $66596 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2418 left
+    we have $2422 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sun Feb 12 17:02:03 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Sun, 12 Feb 2017 14:02:03 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: merged upstream
Message-ID: <58a0db5b.9d711c0a.fa39e.dc46@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r857:0df252ac53a4
Date: 2017-02-12 16:01 -0600
http://bitbucket.org/pypy/pypy.org/changeset/0df252ac53a4/

Log:	merged upstream

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66589 of $105000 (63.4%)
+   $66596 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2417 left
+    we have $2422 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $59040 of $80000 (73.8%)
+   $59050 of $80000 (73.8%)
    <div id="progressbar">
    </div>
 
@@ -29,7 +29,7 @@
   
   <!--
   Current status:
-    we have $48229 left
+    we have $48238 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sun Feb 12 17:02:01 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Sun, 12 Feb 2017 14:02:01 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: Use HTTPS urls for external
 resources
Message-ID: <58a0db59.8b091c0a.af810.dd01@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r856:ce7200d08930
Date: 2017-02-12 16:00 -0600
http://bitbucket.org/pypy/pypy.org/changeset/ce7200d08930/

Log:	Use HTTPS urls for external resources

diff --git a/archive.html b/archive.html
--- a/archive.html
+++ b/archive.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/contact.html b/contact.html
--- a/contact.html
+++ b/contact.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/howtohelp.html b/howtohelp.html
--- a/howtohelp.html
+++ b/howtohelp.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/people.html b/people.html
--- a/people.html
+++ b/people.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/performance.html b/performance.html
--- a/performance.html
+++ b/performance.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi
--- a/source/_layouts/site.genshi
+++ b/source/_layouts/site.genshi
@@ -42,10 +42,10 @@
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css"/>
 	<py:if test="defined('typekit')">
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
 	</py:if>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"/>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/sponsor.html b/sponsor.html
--- a/sponsor.html
+++ b/sponsor.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/success.html b/success.html
--- a/success.html
+++ b/success.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/tmdonate.html b/tmdonate.html
--- a/tmdonate.html
+++ b/tmdonate.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
diff --git a/tmdonate2.html b/tmdonate2.html
--- a/tmdonate2.html
+++ b/tmdonate2.html
@@ -11,9 +11,9 @@
 	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
 	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
   <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
-	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript" src="https://use.typekit.com/hdt8sni.js"></script>
 	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
-	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+	<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>

From pypy.commits at gmail.com  Sun Feb 12 20:17:28 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Sun, 12 Feb 2017 17:17:28 -0800 (PST)
Subject: [pypy-commit] cffi sirtom67/float_complex: all but two tests in
 c/test_c.py:test_complex_types() pass.
Message-ID: <58a10928.56711c0a.91eee.fddd@mx.google.com>

Author: Tom Krauss <thomas.p.krauss at gmail.com>
Branch: sirtom67/float_complex
Changeset: r2889:82ad2ada5811
Date: 2017-02-12 19:16 -0600
http://bitbucket.org/cffi/cffi/changeset/82ad2ada5811/

Log:	all but two tests in c/test_c.py:test_complex_types() pass.

	 1) assert repr(complex(cast(p, -0j))) == '-0j' 2) assert
	complex(cast(p, '\x09')) == 9.0

	I haven't looked at 2) yet but 1) is failing due to python's
	behavior: the answer should be -0j, it is returning 0j instead.

	The issue is in complexobject.c: complex_new, at the end where it
	does this: if (cr_is_complex) { ci.real += cr.imag; } return
	complex_subtype_from_doubles(type, cr.real, ci.real);

	cr.imag is appropriately "-0" still, but after this statement,
	ci.real is just 0. The resulting returned object is just 0j, not
	-0j.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2842,7 +2842,10 @@
 
     if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
         Py_complex value = read_raw_complex_data(cd->c_data, cd->c_type->ct_size);
-        return PyComplex_FromCComplex(value);
+        PyObject *op = PyComplex_FromCComplex(value);
+        PyComplexObject *opc = (PyComplexObject *) op;
+        printf("%f %f\n",opc->cval.real,opc->cval.imag);
+        return op;
     }
     PyErr_Format(PyExc_TypeError, "complex() not supported on cdata '%s'",
                  cd->c_type->ct_name);
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -211,8 +211,8 @@
         assert complex(cast(p, True)) == 1.0
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(cast, new_primitive_type(name), 1+2j)
-    py.test.raises(cast, new_primitive_type("int"), 1+2j)
+        py.test.raises(TypeError, cast, new_primitive_type(name), 1+2j)
+    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+2j)
 
 def test_character_type():
     p = new_primitive_type("char")

From pypy.commits at gmail.com  Mon Feb 13 04:02:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 01:02:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix for error messages
Message-ID: <58a17637.85b2df0a.769d0.4664@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90073:292382c6aeef
Date: 2017-02-13 10:02 +0100
http://bitbucket.org/pypy/pypy/changeset/292382c6aeef/

Log:	fix for error messages

diff --git a/lib-python/3/test/test_complex.py b/lib-python/3/test/test_complex.py
--- a/lib-python/3/test/test_complex.py
+++ b/lib-python/3/test/test_complex.py
@@ -330,11 +330,13 @@
         self.assertRaises(ValueError, complex, ")1+2j(")
         self.assertRaisesRegex(
             TypeError,
-            "first argument must be a string or a number, not 'dict'",
+            "first argument must be a string or a number, not 'dict'"  # cpython
+            "|unsupported operand type for float\(\): 'dict'",         # pypy
             complex, {1:2}, 1)
         self.assertRaisesRegex(
             TypeError,
-            "second argument must be a number, not 'dict'",
+            "second argument must be a number, not 'dict'"          # cpython
+            "|unsupported operand type for float\(\): 'dict'",      # pypy
             complex, 1, {1:2})
         # the following three are accepted by Python 2.6
         self.assertRaises(ValueError, complex, "1..1j")

From pypy.commits at gmail.com  Mon Feb 13 04:14:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 01:14:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip two tests about subinterpreters
Message-ID: <58a1790b.c791df0a.5ae68.3b1c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90074:2f6f33fa5864
Date: 2017-02-13 10:14 +0100
http://bitbucket.org/pypy/pypy/changeset/2f6f33fa5864/

Log:	skip two tests about subinterpreters

diff --git a/lib-python/3/test/test_atexit.py b/lib-python/3/test/test_atexit.py
--- a/lib-python/3/test/test_atexit.py
+++ b/lib-python/3/test/test_atexit.py
@@ -145,6 +145,7 @@
 
 class SubinterpreterTest(unittest.TestCase):
 
+    @support.cpython_only
     def test_callbacks_leak(self):
         # This test shows a leak in refleak mode if atexit doesn't
         # take care to free callbacks in its per-subinterpreter module
@@ -161,6 +162,7 @@
         self.assertEqual(ret, 0)
         self.assertEqual(atexit._ncallbacks(), n)
 
+    @support.cpython_only
     def test_callbacks_leak_refcycle(self):
         # Similar to the above, but with a refcycle through the atexit
         # module.

From pypy.commits at gmail.com  Mon Feb 13 06:00:08 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 13 Feb 2017 03:00:08 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: copy over changes from
 vmprof-python.git repo
Message-ID: <58a191b8.eb88df0a.bd03a.6b09@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90076:4abe2a5167f2
Date: 2017-02-13 11:41 +0100
http://bitbucket.org/pypy/pypy/changeset/4abe2a5167f2/

Log:	copy over changes from vmprof-python.git repo

diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -28,7 +28,7 @@
 
 #ifdef VMPROF_UNIX
 #ifdef __clang__
-__attribute__((disable_tail_calls))
+__attribute__((optnone))
 #elif defined(__GNUC__)
 __attribute__((optimize("O1")))
 #endif
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -164,8 +164,10 @@
         return 0;
     }
 
+#ifdef VMPROF_LINUX
     while (signal) {
-        if (unw_is_signal_frame(&cursor)) {
+        int is_signal_frame = unw_is_signal_frame(&cursor);
+        if (is_signal_frame) {
             break;
         }
         int err = unw_step(&cursor);
@@ -173,6 +175,16 @@
             return 0;
         }
     }
+#else
+    // who would have guessed that unw_is_signal_frame does not work on mac os x
+    if (signal) {
+        unw_step(&cursor); // vmp_walk_and_record_stack
+        // get_stack_trace is inlined
+        unw_step(&cursor); // _vmprof_sample_stack
+        unw_step(&cursor); // sigprof_handler
+        unw_step(&cursor); // _sigtramp
+    }
+#endif
 
     //printf("stack trace:\n");
     int depth = 0;
@@ -265,7 +277,6 @@
             //printf("sample ended\n");
             break;
         } else if (err < 0) {
-            //printf("sample is broken\n");
             return 0; // this sample is broken, cannot walk it fully
         }
     }
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -89,7 +89,7 @@
  * *************************************************************
  */
 
-int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc)
+static int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc)
 {
     PY_STACK_FRAME_T * frame;
 #ifdef RPYTHON_VMPROF
@@ -119,7 +119,7 @@
     longjmp(restore_point, SIGSEGV);
 }
 
-int _vmprof_sample_stack(struct profbuf_s *p, PY_THREAD_STATE_T * tstate, ucontext_t * uc)
+static int _vmprof_sample_stack(struct profbuf_s *p, PY_THREAD_STATE_T * tstate, ucontext_t * uc)
 {
     int depth;
     struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
@@ -312,7 +312,8 @@
     tstate->interp->eval_frame = vmprof_eval;
     _default_eval_loop = _PyEval_EvalFrameDefault;
 #elif defined(RPYTHON_VMPROF)
-    // TODO nothing?
+    // do nothing here, the stack is maintained by rpython
+    // no need for a trampoline
 #else
     if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx,
                 vmprof_eval, (void*)&_default_eval_loop) == 0) {

From pypy.commits at gmail.com  Mon Feb 13 06:00:10 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 13 Feb 2017 03:00:10 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: check that vmprof/vmprof-python
 is in sync with pypy repo (as done for cffi),
 not having this test diverges the two code bases
Message-ID: <58a191ba.025a1c0a.acb48.7907@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90077:f4f98e95445a
Date: 2017-02-13 11:59 +0100
http://bitbucket.org/pypy/pypy/changeset/f4f98e95445a/

Log:	check that vmprof/vmprof-python is in sync with pypy repo (as done
	for cffi), not having this test diverges the two code bases

diff --git a/rpython/rlib/rvmprof/test/test_file.py b/rpython/rlib/rvmprof/test/test_file.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/test/test_file.py
@@ -0,0 +1,26 @@
+import os
+import urllib2, py
+from os.path import join
+
+
+def github_raw_file(repo, path, branch='master'):
+    return "https://raw.githubusercontent.com/{repo}/{branch}/{path}".format(**dict(
+                repo=repo, path=path, branch=branch
+            ))
+
+
+def test_same_file():
+    for root, dirs, files in os.walk('rpython/rlib/rvmprof/src/shared'):
+        for file in files:
+            if not (file.endswith(".c") or file.endswith(".h")):
+                continue
+            url = github_raw_file("vmprof/vmprof-python", "src/%s" % file)
+            source = urllib2.urlopen(url).read()
+            #
+            dest = py.path.local(join(root, file)).read()
+            if source != dest:
+                raise AssertionError("%s was updated, but changes were"
+                                     "not copied over to PyPy" % url)
+            else:
+                print("%s matches" % url)
+        break # do not walk dirs

From pypy.commits at gmail.com  Mon Feb 13 06:00:06 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 13 Feb 2017 03:00:06 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: local variable overwrote previous
 definition, crashes translated version
Message-ID: <58a191b6.c53f1c0a.1529e.7d72@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90075:38a11c8364c1
Date: 2017-02-13 11:38 +0100
http://bitbucket.org/pypy/pypy/changeset/38a11c8364c1/

Log:	local variable overwrote previous definition, crashes translated
	version

diff --git a/rpython/jit/metainterp/optimizeopt/version.py b/rpython/jit/metainterp/optimizeopt/version.py
--- a/rpython/jit/metainterp/optimizeopt/version.py
+++ b/rpython/jit/metainterp/optimizeopt/version.py
@@ -76,7 +76,7 @@
                                                  descr, vl.inputargs,
                                                  vl.operations, jitcell_token,
                                                  metainterp.box_names_memo)
-                record_loop_or_bridge(metainterp_sd, vl)
+                record_loop_or_bridge(metainterp_sd, vl, asminfo)
                 assert asminfo is not None
                 compiled[version] = (asminfo, descr, version, jitcell_token)
             else:
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -106,9 +106,9 @@
                                   lltype.Void, compilation_info=eci,
                                   _nowrapper=True)
 
-    vmp_dyn_cancel = rffi.llexternal("vmp_dyn_teardown", [lltype.Void],
-                                  rffi.INT, compilation_info=eci,
-                                  _nowrapper=True)
+    vmp_dyn_teardown = rffi.llexternal("vmp_dyn_teardown", [lltype.Void],
+                                       rffi.INT, compilation_info=eci,
+                                       _nowrapper=True)
 
     return CInterface(locals())
 

From pypy.commits at gmail.com  Mon Feb 13 06:42:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 03:42:24 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: hg merge default
Message-ID: <58a19ba0.018e1c0a.e61b9.879a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90078:7ae20dbc22fb
Date: 2017-02-13 11:34 +0100
http://bitbucket.org/pypy/pypy/changeset/7ae20dbc22fb/

Log:	hg merge default

diff too long, truncating to 2000 out of 117896 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/ctypes_configure/__init__.py b/ctypes_configure/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/cbuild.py b/ctypes_configure/cbuild.py
deleted file mode 100644
--- a/ctypes_configure/cbuild.py
+++ /dev/null
@@ -1,456 +0,0 @@
-
-import os, sys, inspect, re, imp, py
-from ctypes_configure import stdoutcapture
-import distutils
-
-debug = 0
-
-configdir = py.path.local.make_numbered_dir(prefix='ctypes_configure-')
-
-class ExternalCompilationInfo(object):
-
-    _ATTRIBUTES = ['pre_include_lines', 'includes', 'include_dirs',
-                   'post_include_lines', 'libraries', 'library_dirs',
-                   'separate_module_sources', 'separate_module_files']
-    _AVOID_DUPLICATES = ['separate_module_files', 'libraries', 'includes',
-                         'include_dirs', 'library_dirs', 'separate_module_sources']
-
-    def __init__(self,
-                 pre_include_lines       = [],
-                 includes                = [],
-                 include_dirs            = [],
-                 post_include_lines      = [],
-                 libraries               = [],
-                 library_dirs            = [],
-                 separate_module_sources = [],
-                 separate_module_files   = []):
-        """
-        pre_include_lines: list of lines that should be put at the top
-        of the generated .c files, before any #include.  They shouldn't
-        contain an #include themselves.
-
-        includes: list of .h file names to be #include'd from the
-        generated .c files.
-
-        include_dirs: list of dir names that is passed to the C compiler
-
-        post_include_lines: list of lines that should be put at the top
-        of the generated .c files, after the #includes.
-
-        libraries: list of library names that is passed to the linker
-
-        library_dirs: list of dir names that is passed to the linker
-
-        separate_module_sources: list of multiline strings that are
-        each written to a .c file and compiled separately and linked
-        later on.  (If function prototypes are needed for other .c files
-        to access this, they can be put in post_include_lines.)
-
-        separate_module_files: list of .c file names that are compiled
-        separately and linked later on.  (If an .h file is needed for
-        other .c files to access this, it can be put in includes.)
-        """
-        for name in self._ATTRIBUTES:
-            value = locals()[name]
-            assert isinstance(value, (list, tuple))
-            setattr(self, name, tuple(value))
-
-    def _value(self):
-        return tuple([getattr(self, x) for x in self._ATTRIBUTES])
-
-    def __hash__(self):
-        return hash(self._value())
-
-    def __eq__(self, other):
-        return self.__class__ is other.__class__ and \
-               self._value() == other._value()
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __repr__(self):
-        info = []
-        for attr in self._ATTRIBUTES:
-            val = getattr(self, attr)
-            info.append("%s=%s" % (attr, repr(val)))
-        return "<ExternalCompilationInfo (%s)>" % ", ".join(info)
-
-    def merge(self, *others):
-        others = list(others)
-        attrs = {}
-        for name in self._ATTRIBUTES:
-            if name not in self._AVOID_DUPLICATES:
-                s = []
-                for i in [self] + others:
-                    s += getattr(i, name)
-                attrs[name] = s
-            else:
-                s = set()
-                attr = []
-                for one in [self] + others:
-                    for elem in getattr(one, name):
-                        if elem not in s:
-                            s.add(elem)
-                            attr.append(elem)
-                attrs[name] = attr
-        return ExternalCompilationInfo(**attrs)
-
-    def write_c_header(self, fileobj):
-        for line in self.pre_include_lines:
-            print >> fileobj, line
-        for path in self.includes:
-            print >> fileobj, '#include <%s>' % (path,)
-        for line in self.post_include_lines:
-            print >> fileobj, line
-
-    def _copy_attributes(self):
-        d = {}
-        for attr in self._ATTRIBUTES:
-            d[attr] = getattr(self, attr)
-        return d
-
-    def convert_sources_to_files(self, cache_dir=None, being_main=False):
-        if not self.separate_module_sources:
-            return self
-        if cache_dir is None:
-            cache_dir = configdir.join('module_cache').ensure(dir=1)
-        num = 0
-        files = []
-        for source in self.separate_module_sources:
-            while 1:
-                filename = cache_dir.join('module_%d.c' % num)
-                num += 1
-                if not filename.check():
-                    break
-            f = filename.open("w")
-            if being_main:
-                f.write("#define PYPY_NOT_MAIN_FILE\n")
-            self.write_c_header(f)
-            source = str(source)
-            f.write(source)
-            if not source.endswith('\n'):
-                f.write('\n')
-            f.close()
-            files.append(str(filename))
-        d = self._copy_attributes()
-        d['separate_module_sources'] = ()
-        d['separate_module_files'] += tuple(files)
-        return ExternalCompilationInfo(**d)
-
-    def compile_shared_lib(self):
-        self = self.convert_sources_to_files()
-        if not self.separate_module_files:
-            return self
-        lib = compile_c_module([], 'externmod', self)
-        d = self._copy_attributes()
-        d['libraries'] += (lib,)
-        d['separate_module_files'] = ()
-        d['separate_module_sources'] = ()
-        return ExternalCompilationInfo(**d)
-
-if sys.platform == 'win32':
-    so_ext = '.dll'
-else:
-    so_ext = '.so'
-
-def compiler_command():
-    # e.g. for tcc, you might set this to
-    #    "tcc -shared -o %s.so %s.c"
-    return os.getenv('PYPY_CC')
-
-def enable_fast_compilation():
-    if sys.platform == 'win32':
-        dash = '/'
-    else:
-        dash = '-'
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt:
-        opt = re.sub('%sO\d+' % dash, '%sO0' % dash, opt)
-    else:
-        opt = '%sO0' % dash
-    gcv['OPT'] = opt
-
-def ensure_correct_math():
-    if sys.platform != 'win32':
-        return # so far
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt and '/Op' not in opt:
-        opt += '/Op'
-    gcv['OPT'] = opt
-
-
-def try_compile(c_files, eci):
-    try:
-        build_executable(c_files, eci)
-        result = True
-    except (distutils.errors.CompileError,
-            distutils.errors.LinkError):
-        result = False
-    return result
-
-def compile_c_module(cfiles, modbasename, eci, tmpdir=None):
-    #try:
-    #    from distutils.log import set_threshold
-    #    set_threshold(10000)
-    #except ImportError:
-    #    print "ERROR IMPORTING"
-    #    pass
-    cfiles = [py.path.local(f) for f in cfiles]
-    if tmpdir is None:
-        tmpdir = configdir.join("module_cache").ensure(dir=1)
-    num = 0
-    cfiles += eci.separate_module_files
-    include_dirs = list(eci.include_dirs)
-    library_dirs = list(eci.library_dirs)
-    if (sys.platform == 'darwin' or    # support Fink & Darwinports
-            sys.platform.startswith('freebsd')):
-        for s in ('/sw/', '/opt/local/', '/usr/local/'):
-            if s + 'include' not in include_dirs and \
-               os.path.exists(s + 'include'):
-                include_dirs.append(s + 'include')
-            if s + 'lib' not in library_dirs and \
-               os.path.exists(s + 'lib'):
-                library_dirs.append(s + 'lib')
-
-    num = 0
-    modname = modbasename
-    while 1:
-        if not tmpdir.join(modname + so_ext).check():
-            break
-        num += 1
-        modname = '%s_%d' % (modbasename, num)
-
-    lastdir = tmpdir.chdir()
-    libraries = eci.libraries
-    ensure_correct_math()
-    try:
-        if debug: print "modname", modname
-        c = stdoutcapture.Capture(mixed_out_err = True)
-        try:
-            try:
-                if compiler_command():
-                    # GCC-ish options only
-                    from distutils import sysconfig
-                    gcv = sysconfig.get_config_vars()
-                    cmd = compiler_command().replace('%s',
-                                                     str(tmpdir.join(modname)))
-                    for dir in [gcv['INCLUDEPY']] + list(include_dirs):
-                        cmd += ' -I%s' % dir
-                    for dir in library_dirs:
-                        cmd += ' -L%s' % dir
-                    os.system(cmd)
-                else:
-                    from distutils.dist import Distribution
-                    from distutils.extension import Extension
-                    from distutils.ccompiler import get_default_compiler
-                    saved_environ = os.environ.items()
-                    try:
-                        # distutils.core.setup() is really meant for end-user
-                        # interactive usage, because it eats most exceptions and
-                        # turn them into SystemExits.  Instead, we directly
-                        # instantiate a Distribution, which also allows us to
-                        # ignore unwanted features like config files.
-                        extra_compile_args = []
-                        # ensure correct math on windows
-                        if sys.platform == 'win32':
-                            extra_compile_args.append('/Op') # get extra precision
-                        if get_default_compiler() == 'unix':
-                            old_version = False
-                            try:
-                                g = os.popen('gcc --version', 'r')
-                                verinfo = g.read()
-                                g.close()
-                            except (OSError, IOError):
-                                pass
-                            else:
-                                old_version = verinfo.startswith('2')
-                            if not old_version:
-                                extra_compile_args.extend(["-Wno-unused-label",
-                                                        "-Wno-unused-variable"])
-                        attrs = {
-                            'name': "testmodule",
-                            'ext_modules': [
-                                Extension(modname, [str(cfile) for cfile in cfiles],
-                                    include_dirs=include_dirs,
-                                    library_dirs=library_dirs,
-                                    extra_compile_args=extra_compile_args,
-                                    libraries=list(libraries),)
-                                ],
-                            'script_name': 'setup.py',
-                            'script_args': ['-q', 'build_ext', '--inplace', '--force'],
-                            }
-                        dist = Distribution(attrs)
-                        if not dist.parse_command_line():
-                            raise ValueError, "distutils cmdline parse error"
-                        dist.run_commands()
-                    finally:
-                        for key, value in saved_environ:
-                            if os.environ.get(key) != value:
-                                os.environ[key] = value
-            finally:
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = open("%s.errors" % modname, "w")
-                    fdump.write(data)
-                    fdump.close()
-            # XXX do we need to do some check on fout/ferr?
-            # XXX not a nice way to import a module
-        except:
-            print >>sys.stderr, data
-            raise
-    finally:
-        lastdir.chdir()
-    return str(tmpdir.join(modname) + so_ext)
-
-def make_module_from_c(cfile, eci):
-    cfile = py.path.local(cfile)
-    modname = cfile.purebasename
-    compile_c_module([cfile], modname, eci)
-    return import_module_from_directory(cfile.dirpath(), modname)
-
-def import_module_from_directory(dir, modname):
-    file, pathname, description = imp.find_module(modname, [str(dir)])
-    try:
-        mod = imp.load_module(modname, file, pathname, description)
-    finally:
-        if file:
-            file.close()
-    return mod
-
-
-def log_spawned_cmd(spawn):
-    def spawn_and_log(cmd, *args, **kwds):
-        if debug:
-            print ' '.join(cmd)
-        return spawn(cmd, *args, **kwds)
-    return spawn_and_log
-
-
-class ProfOpt(object):
-    #XXX assuming gcc style flags for now
-    name = "profopt"
-    
-    def __init__(self, compiler):
-        self.compiler = compiler
-
-    def first(self):
-        self.build('-fprofile-generate')
-
-    def probe(self, exe, args):
-        # 'args' is a single string typically containing spaces
-        # and quotes, which represents several arguments.
-        os.system("'%s' %s" % (exe, args))
-
-    def after(self):
-        self.build('-fprofile-use')
-
-    def build(self, option):
-        compiler = self.compiler
-        compiler.compile_extra.append(option)
-        compiler.link_extra.append(option)
-        try:
-            compiler._build()
-        finally:
-            compiler.compile_extra.pop()
-            compiler.link_extra.pop()
-            
-class CCompiler:
-
-    def __init__(self, cfilenames, eci, outputfilename=None,
-                 compiler_exe=None, profbased=None):
-        self.cfilenames = cfilenames
-        ext = ''
-        self.compile_extra = []
-        self.link_extra = []
-        self.libraries = list(eci.libraries)
-        self.include_dirs = list(eci.include_dirs)
-        self.library_dirs = list(eci.library_dirs)
-        self.compiler_exe = compiler_exe
-        self.profbased = profbased
-        if not sys.platform in ('win32', 'darwin', 'cygwin'): # xxx
-            if 'm' not in self.libraries:
-                self.libraries.append('m')
-            if 'pthread' not in self.libraries:
-                self.libraries.append('pthread')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer', '-pthread']
-            self.link_extra += ['-pthread']
-        if sys.platform == 'win32':
-            self.link_extra += ['/DEBUG'] # generate .pdb file
-        if (sys.platform == 'darwin' or    # support Fink & Darwinports
-                sys.platform.startswith('freebsd')):
-            for s in ('/sw/', '/opt/local/', '/usr/local/'):
-                if s + 'include' not in self.include_dirs and \
-                   os.path.exists(s + 'include'):
-                    self.include_dirs.append(s + 'include')
-                if s + 'lib' not in self.library_dirs and \
-                   os.path.exists(s + 'lib'):
-                    self.library_dirs.append(s + 'lib')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer']
-
-        if outputfilename is None:
-            self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext)
-        else: 
-            self.outputfilename = py.path.local(outputfilename)
-
-    def build(self, noerr=False):
-        basename = self.outputfilename.new(ext='')
-        data = ''
-        try:
-            saved_environ = os.environ.copy()
-            c = stdoutcapture.Capture(mixed_out_err = True)
-            try:
-                self._build()
-            finally:
-                # workaround for a distutils bugs where some env vars can
-                # become longer and longer every time it is used
-                for key, value in saved_environ.items():
-                    if os.environ.get(key) != value:
-                        os.environ[key] = value
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = basename.new(ext='errors').open("w")
-                    fdump.write(data)
-                    fdump.close()
-        except:
-            if not noerr:
-                print >>sys.stderr, data
-            raise
- 
-    def _build(self):
-        from distutils.ccompiler import new_compiler 
-        compiler = new_compiler(force=1)
-        if self.compiler_exe is not None:
-            for c in '''compiler compiler_so compiler_cxx
-                        linker_exe linker_so'''.split():
-                compiler.executables[c][0] = self.compiler_exe
-        compiler.spawn = log_spawned_cmd(compiler.spawn)
-        objects = []
-        for cfile in self.cfilenames: 
-            cfile = py.path.local(cfile)
-            old = cfile.dirpath().chdir() 
-            try: 
-                res = compiler.compile([cfile.basename], 
-                                       include_dirs=self.include_dirs,
-                                       extra_preargs=self.compile_extra)
-                assert len(res) == 1
-                cobjfile = py.path.local(res[0]) 
-                assert cobjfile.check()
-                objects.append(str(cobjfile))
-            finally: 
-                old.chdir() 
-        compiler.link_executable(objects, str(self.outputfilename),
-                                 libraries=self.libraries,
-                                 extra_preargs=self.link_extra,
-                                 library_dirs=self.library_dirs)
-
-def build_executable(*args, **kwds):
-    noerr = kwds.pop('noerr', False)
-    compiler = CCompiler(*args, **kwds)
-    compiler.build(noerr=noerr)
-    return str(compiler.outputfilename)
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
deleted file mode 100755
--- a/ctypes_configure/configure.py
+++ /dev/null
@@ -1,621 +0,0 @@
-#! /usr/bin/env python
-
-import os, py, sys
-import ctypes
-from ctypes_configure.cbuild import build_executable, configdir, try_compile
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import distutils
-
-# ____________________________________________________________
-#
-# Helpers for simple cases
-
-def eci_from_header(c_header_source):
-    return ExternalCompilationInfo(
-        pre_include_lines=c_header_source.split("\n")
-    )
-
-
-def getstruct(name, c_header_source, interesting_fields):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        STRUCT = Struct(name, interesting_fields)
-    return configure(CConfig)['STRUCT']
-
-def getsimpletype(name, c_header_source, ctype_hint=ctypes.c_int):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        TYPE = SimpleType(name, ctype_hint)
-    return configure(CConfig)['TYPE']
-
-def getconstantinteger(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        CONST = ConstantInteger(name)
-    return configure(CConfig)['CONST']
-
-def getdefined(macro, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        DEFINED = Defined(macro)
-    return configure(CConfig)['DEFINED']
-
-def has(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        HAS = Has(name)
-    return configure(CConfig)['HAS']
-
-def check_eci(eci):
-    """Check if a given ExternalCompilationInfo compiles and links."""
-    class CConfig:
-        _compilation_info_ = eci
-        WORKS = Works()
-    return configure(CConfig)['WORKS']
-
-def sizeof(name, eci, **kwds):
-    class CConfig:
-        _compilation_info_ = eci
-        SIZE = SizeOf(name)
-    for k, v in kwds.items():
-        setattr(CConfig, k, v)
-    return configure(CConfig)['SIZE']
-
-def memory_alignment():
-    """Return the alignment (in bytes) of memory allocations.
-    This is enough to make sure a structure with pointers and 'double'
-    fields is properly aligned."""
-    global _memory_alignment
-    if _memory_alignment is None:
-        S = getstruct('struct memory_alignment_test', """
-           struct memory_alignment_test {
-               double d;
-               void* p;
-           };
-        """, [])
-        result = ctypes.alignment(S)
-        assert result & (result-1) == 0, "not a power of two??"
-        _memory_alignment = result
-    return _memory_alignment
-_memory_alignment = None
-
-# ____________________________________________________________
-#
-# General interface
-
-class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
-        self.result = {}
-        self.info = info
-        self.entries = entries
-        
-    def get_entry_result(self, entry):
-        try:
-            return self.result[entry]
-        except KeyError:
-            pass
-        name = self.entries[entry]
-        info = self.info[name]
-        self.result[entry] = entry.build_result(info, self)
-
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
-
-
-class _CWriter(object):
-    """ A simple class which aggregates config parts
-    """
-    def __init__(self, CConfig):
-        self.path = uniquefilepath()
-        self.f = self.path.open("w")
-        self.config = CConfig
-
-    def write_header(self):
-        f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
-        print >> f, C_HEADER
-        print >> f
-
-    def write_entry(self, key, entry):
-        f = self.f
-        print >> f, 'void dump_section_%s(void) {' % (key,)
-        for line in entry.prepare_code():
-            if line and line[0] != '#':
-                line = '\t' + line
-            print >> f, line
-        print >> f, '}'
-        print >> f
-
-    def write_entry_main(self, key):
-        print >> self.f, '\tprintf("-+- %s\\n");' % (key,)
-        print >> self.f, '\tdump_section_%s();' % (key,)
-        print >> self.f, '\tprintf("---\\n");'
-
-    def start_main(self):
-        print >> self.f, 'int main(int argc, char *argv[]) {'
-
-    def close(self):
-        f = self.f
-        print >> f, '\treturn 0;'
-        print >> f, '}'
-        f.close()
-
-    def ask_gcc(self, question):
-        self.start_main()
-        self.f.write(question + "\n")
-        self.close()
-        eci = self.config._compilation_info_
-        return try_compile([self.path], eci)
-
-        
-def configure(CConfig, noerr=False):
-    """Examine the local system by running the C compiler.
-    The CConfig class contains CConfigEntry attribues that describe
-    what should be inspected; configure() returns a dict mapping
-    names to the results.
-    """
-    for attr in ['_includes_', '_libraries_', '_sources_', '_library_dirs_',
-                 '_include_dirs_', '_header_']:
-        assert not hasattr(CConfig, attr), "Found legacy attribut %s on CConfig" % (attr,)
-    entries = []
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigEntry):
-            entries.append((key, value))
-
-    if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        f = writer.f
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci, noerr=noerr))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(CConfig, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
-    else:
-        res = {}
-
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
-            writer.write_header()
-            res[key] = value.question(writer.ask_gcc)
-    return res
-
-# ____________________________________________________________
-
-
-class CConfigEntry(object):
-    "Abstract base class."
-
-class Struct(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined structure.
-    """
-    def __init__(self, name, interesting_fields, ifdef=None):
-        self.name = name
-        self.interesting_fields = interesting_fields
-        self.ifdef = ifdef
-
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield 'typedef struct {'
-        yield '    char c;'
-        yield '    ctypesplatcheck_t s;'
-        yield '} ctypesplatcheck2_t;'
-        yield ''
-        yield 'ctypesplatcheck_t s;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("align", offsetof(ctypesplatcheck2_t, s));'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        for fieldname, fieldtype in self.interesting_fields:
-            yield 'dump("fldofs %s", offsetof(ctypesplatcheck_t, %s));'%(
-                fieldname, fieldname)
-            yield 'dump("fldsize %s",   sizeof(s.%s));' % (
-                fieldname, fieldname)
-            if fieldtype in integer_class:
-                yield 's.%s = 0; s.%s = ~s.%s;' % (fieldname,
-                                                   fieldname,
-                                                   fieldname)
-                yield 'dump("fldunsigned %s", s.%s > 0);' % (fieldname,
-                                                             fieldname)
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None:
-            if not info['defined']:
-                return None
-        alignment = 1
-        layout = [None] * info['size']
-        for fieldname, fieldtype in self.interesting_fields:
-            if isinstance(fieldtype, Struct):
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                c_fieldtype = config_result.get_entry_result(fieldtype)
-                layout_addfield(layout, offset, c_fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(c_fieldtype))
-            else:
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                sign   = info.get('fldunsigned ' + fieldname, False)
-                if (size, sign) != size_and_sign(fieldtype):
-                    fieldtype = fixup_ctype(fieldtype, fieldname, (size, sign))
-                layout_addfield(layout, offset, fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(fieldtype))
-
-        # try to enforce the same alignment as the one of the original
-        # structure
-        if alignment < info['align']:
-            choices = [ctype for ctype in alignment_types
-                             if ctype_alignment(ctype) == info['align']]
-            assert choices, "unsupported alignment %d" % (info['align'],)
-            choices = [(ctypes.sizeof(ctype), i, ctype)
-                       for i, ctype in enumerate(choices)]
-            csize, _, ctype = min(choices)
-            for i in range(0, info['size'] - csize + 1, info['align']):
-                if layout[i:i+csize] == [None] * csize:
-                    layout_addfield(layout, i, ctype, '_alignment')
-                    break
-            else:
-                raise AssertionError("unenforceable alignment %d" % (
-                    info['align'],))
-
-        n = 0
-        for i, cell in enumerate(layout):
-            if cell is not None:
-                continue
-            layout_addfield(layout, i, ctypes.c_char, '_pad%d' % (n,))
-            n += 1
-
-        # build the ctypes Structure
-        seen = {}
-        fields = []
-        for cell in layout:
-            if cell in seen:
-                continue
-            fields.append((cell.name, cell.ctype))
-            seen[cell] = True
-
-        class S(ctypes.Structure):
-            _fields_ = fields
-        name = self.name
-        if name.startswith('struct '):
-            name = name[7:]
-        S.__name__ = name
-        return S
-
-class SimpleType(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined simple numeric type.
-    """
-    def __init__(self, name, ctype_hint=ctypes.c_int, ifdef=None):
-        self.name = name
-        self.ctype_hint = ctype_hint
-        self.ifdef = ifdef
-        
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield ''
-        yield 'ctypesplatcheck_t x;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        if self.ctype_hint in integer_class:
-            yield 'x = 0; x = ~x;'
-            yield 'dump("unsigned", x > 0);'
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None and not info['defined']:
-            return None
-        size = info['size']
-        sign = info.get('unsigned', False)
-        ctype = self.ctype_hint
-        if (size, sign) != size_and_sign(ctype):
-            ctype = fixup_ctype(ctype, self.name, (size, sign))
-        return ctype
-
-class ConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant.
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
-        yield '    long long x = (long long)(%s);' % (self.name,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.name,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-
-    def build_result(self, info, config_result):
-        return info['value']
-
-class DefinedConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant. If not #defined the value will be None.
-    """
-    def __init__(self, macro):
-        self.name = self.macro = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
-        yield '    long long x = (long long)(%s);' % (self.macro,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.macro,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            return info['value']
-        return None
-
-
-class DefinedConstantString(CConfigEntry):
-    """
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'int i;'
-        yield 'char *p = %s;' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'for (i = 0; p[i] != 0; i++ ) {'
-        yield '  printf("value_%d: %d\\n", i, (int)(unsigned char)p[i]);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            string = ''
-            d = 0
-            while info.has_key('value_%d' % d):
-                string += chr(info['value_%d' % d])
-                d += 1
-            return string
-        return None
-
-
-class Defined(CConfigEntry):
-    """A boolean, corresponding to an #ifdef.
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % (self.macro,)
-        yield 'dump("defined", 1);'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        return bool(info['defined'])
-
-class CConfigSingleEntry(object):
-    """ An abstract class of type which requires
-    gcc succeeding/failing instead of only asking
-    """
-    pass
-
-class Has(CConfigSingleEntry):
-    def __init__(self, name):
-        self.name = name
-    
-    def question(self, ask_gcc):
-        return ask_gcc(self.name + ';')
-
-class Works(CConfigSingleEntry):
-    def question(self, ask_gcc):
-        return ask_gcc("")
-
-class SizeOf(CConfigEntry):
-    """An entry in a CConfig class that stands for
-    some external opaque type
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'dump("size",  sizeof(%s));' % self.name
-
-    def build_result(self, info, config_result):
-        return info['size']
-
-# ____________________________________________________________
-#
-# internal helpers
-
-def ctype_alignment(c_type):
-    if issubclass(c_type, ctypes.Structure):
-        return max([ctype_alignment(fld_type)
-                     for fld_name, fld_type in c_type._fields_])
-    
-    return ctypes.alignment(c_type)
-
-def uniquefilepath(LAST=[0]):
-    i = LAST[0]
-    LAST[0] += 1
-    return configdir.join('ctypesplatcheck_%d.c' % i)
-
-alignment_types = [
-    ctypes.c_short,
-    ctypes.c_int,
-    ctypes.c_long,
-    ctypes.c_float,
-    ctypes.c_double,
-    ctypes.c_char_p,
-    ctypes.c_void_p,
-    ctypes.c_longlong,
-    ctypes.c_wchar,
-    ctypes.c_wchar_p,
-    ]
-
-integer_class = [ctypes.c_byte,     ctypes.c_ubyte,
-                 ctypes.c_short,    ctypes.c_ushort,
-                 ctypes.c_int,      ctypes.c_uint,
-                 ctypes.c_long,     ctypes.c_ulong,
-                 ctypes.c_longlong, ctypes.c_ulonglong,
-                 ]
-float_class = [ctypes.c_float, ctypes.c_double]
-
-class Field(object):
-    def __init__(self, name, ctype):
-        self.name = name
-        self.ctype = ctype
-    def __repr__(self):
-        return '<field %s: %s>' % (self.name, self.ctype)
-
-def layout_addfield(layout, offset, ctype, prefix):
-    size = ctypes.sizeof(ctype)
-    name = prefix
-    i = 0
-    while name in layout:
-        i += 1
-        name = '%s_%d' % (prefix, i)
-    field = Field(name, ctype)
-    for i in range(offset, offset+size):
-        assert layout[i] is None, "%s overlaps %r" % (fieldname, layout[i])
-        layout[i] = field
-    return field
-
-def size_and_sign(ctype):
-    return (ctypes.sizeof(ctype),
-            ctype in integer_class and ctype(-1).value > 0)
-
-def fixup_ctype(fieldtype, fieldname, expected_size_and_sign):
-    for typeclass in [integer_class, float_class]:
-        if fieldtype in typeclass:
-            for ctype in typeclass:
-                if size_and_sign(ctype) == expected_size_and_sign:
-                    return ctype
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_char):
-        # for now, assume it is an array of chars; otherwise we'd also
-        # have to check the exact integer type of the elements of the array
-        size, sign = expected_size_and_sign
-        return ctypes.c_char * size
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_ubyte):
-        # grumble, fields of type 'c_char array' have automatic cast-to-
-        # Python-string behavior in ctypes, which may not be what you
-        # want, so here is the same with c_ubytes instead...
-        size, sign = expected_size_and_sign
-        return ctypes.c_ubyte * size
-    raise TypeError("conflicting field type %r for %r" % (fieldtype,
-                                                          fieldname))
-
-
-C_HEADER = """
-#include <stdio.h>
-#include <stddef.h>   /* for offsetof() */
-#ifndef _WIN32
-#  include <stdint.h>   /* FreeBSD: for uint64_t */
-#endif
-
-void dump(char* key, int value) {
-    printf("%s: %d\\n", key, value);
-}
-"""
-
-def run_example_code(filepath, eci, noerr=False):
-    executable = build_executable([filepath], eci, noerr=noerr)
-    output = py.process.cmdexec(executable)
-    section = None
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith('-+- '):      # start of a new section
-            section = {}
-        elif line == '---':              # section end
-            assert section is not None
-            yield section
-            section = None
-        elif line:
-            assert section is not None
-            key, value = line.split(': ')
-            section[key] = int(value)
-
-# ____________________________________________________________
-
-def get_python_include_dir():
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    return gcv['INCLUDEPY']
-
-if __name__ == '__main__':
-    doc = """Example:
-    
-       ctypes_platform.py  -h sys/types.h  -h netinet/in.h
-                           'struct sockaddr_in'
-                           sin_port  c_int
-    """
-    import sys, getopt
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'h:')
-    if not args:
-        print >> sys.stderr, doc
-    else:
-        assert len(args) % 2 == 1
-        headers = []
-        for opt, value in opts:
-            if opt == '-h':
-                headers.append('#include <%s>' % (value,))
-        name = args[0]
-        fields = []
-        for i in range(1, len(args), 2):
-            ctype = getattr(ctypes, args[i+1])
-            fields.append((args[i], ctype))
-
-        S = getstruct(name, '\n'.join(headers), fields)
-
-        for key, value in S._fields_:
-            print key, value
diff --git a/ctypes_configure/doc/configure.html b/ctypes_configure/doc/configure.html
deleted file mode 100644
--- a/ctypes_configure/doc/configure.html
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="latin1" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=latin1" />
-<meta name="generator" content="Docutils 0.4.1: http://docutils.sourceforge.net/" />
-<title>ctypes configure</title>
-</head>
-<body>
-<div class="document" id="ctypes-configure">
-<h1 class="title">ctypes configure</h1>
-<div class="section">
-<h1><a id="idea" name="idea">idea</a></h1>
-<p>One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).</p>
-</div>
-<div class="section">
-<h1><a id="installation" name="installation">installation</a></h1>
-<p><tt class="docutils literal"><span class="pre">easy_install</span> <span class="pre">ctypes_configure</span></tt></p>
-</div>
-<div class="section">
-<h1><a id="usage" name="usage">usage</a></h1>
-<p><a class="reference" href="http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py">sample.py</a> explains in details how to use it.</p>
-</div>
-</div>
-</body>
-</html>
diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt
deleted file mode 100644
--- a/ctypes_configure/doc/configure.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-=================
-ctypes configure
-=================
-
-idea
-====
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size\_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).
-
-installation
-============
-
-``easy_install ctypes_configure``
-
-usage
-=====
-
-:source:`sample.py <ctypes_configure/doc/sample.py>` explains in details how to use it.
diff --git a/ctypes_configure/doc/sample.py b/ctypes_configure/doc/sample.py
deleted file mode 100644
--- a/ctypes_configure/doc/sample.py
+++ /dev/null
@@ -1,72 +0,0 @@
-
-from ctypes_configure import configure
-import ctypes
-
-class CConfigure:
-    _compilation_info_ = configure.ExternalCompilationInfo(
-        
-        # all lines landing in C header before includes
-        pre_include_lines = [],
-
-        # list of .h files to include
-        includes = ['time.h', 'sys/time.h', 'unistd.h'],
-
-        # list of directories to search for include files
-        include_dirs = [],
-
-        # all lines landing in C header after includes
-        post_include_lines = [],
-
-        # libraries to link with
-        libraries = [],
-
-        # library directories
-        library_dirs = [],
-
-        # additional C sources to compile with (that go to
-        # created .c files)
-        separate_module_sources = [],
-
-        # additional existing C source file names
-        separate_module_files = [],
-        )
-
-    # get real int type out of hint and name
-    size_t = configure.SimpleType('size_t', ctypes.c_int)
-
-    # grab value of numerical #define
-    NULL = configure.ConstantInteger('NULL')
-
-    # grab #define, whether it's defined or not
-    EXISTANT = configure.Defined('NULL')
-    NOT_EXISTANT = configure.Defined('XXXNOTNULL')
-
-    # check for existance of C functions
-    has_write = configure.Has('write')
-    no_xxxwrite = configure.Has('xxxwrite')
-
-    # check for size of type
-    sizeof_size_t = configure.SizeOf('size_t')
-
-    # structure, with given hints for interesting fields,
-    # types does not need to be too specific.
-    # all interesting fields would end up with right offset
-    # size and order
-    struct_timeval = configure.Struct('struct timeval',[
-        ('tv_sec', ctypes.c_int),
-        ('tv_usec', ctypes.c_int)])
-
-info = configure.configure(CConfigure)
-
-assert info['has_write']
-assert not info['no_xxxwrite']
-assert info['NULL'] == 0
-size_t = info['size_t']
-print "size_t in ctypes is ", size_t
-assert ctypes.sizeof(size_t) == info['sizeof_size_t']
-assert info['EXISTANT']
-assert not info['NOT_EXISTANT']
-print
-print "fields of struct timeval are "
-for name, value in info['struct_timeval']._fields_:
-    print "  ", name, " ", value
diff --git a/ctypes_configure/dumpcache.py b/ctypes_configure/dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/dumpcache.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os, sys
-import ctypes
-
-
-def dumpcache(referencefilename, filename, config):
-    dirname = os.path.dirname(referencefilename)
-    filename = os.path.join(dirname, filename)
-    f = open(filename, 'w')
-    print >> f, 'import ctypes'
-    print >> f
-    names = config.keys()
-    names.sort()
-    print >> f, '__all__ = %r' % (tuple(names),)
-    print >> f
-    for key in names:
-        val = config[key]
-        if isinstance(val, (int, long)):
-            f.write("%s = %d\n" % (key, val))
-        elif val is None:
-            f.write("%s = None\n" % key)
-        elif isinstance(val, ctypes.Structure.__class__):
-            f.write("class %s(ctypes.Structure):\n" % key)
-            f.write("    _fields_ = [\n")
-            for k, v in val._fields_:
-                f.write("        ('%s', %s),\n" % (k, ctypes_repr(v)))
-            f.write("    ]\n")
-        elif isinstance(val, (tuple, list)):
-            for x in val:
-                assert isinstance(x, (int, long, str)), \
-                       "lists of integers or strings only"
-            f.write("%s = %r\n" % (key, val))
-        else:
-            # a simple type, hopefully
-            f.write("%s = %s\n" % (key, ctypes_repr(val)))
-    f.close()
-    print 'Wrote %s.' % (filename,)
-    sys.stdout.flush()
-
-def ctypes_repr(cls):
-    # ctypes_configure does not support nested structs so far
-    # so let's ignore it
-    if isinstance(cls, ctypes._SimpleCData.__class__):
-        return "ctypes." + cls.__name__
-    if hasattr(cls, '_length_') and hasattr(cls, '_type_'):  # assume an array
-        return '%s*%d' % (ctypes_repr(cls._type_), cls._length_)
-    raise NotImplementedError("saving of object with type %r" % type(cls))
diff --git a/ctypes_configure/stdoutcapture.py b/ctypes_configure/stdoutcapture.py
deleted file mode 100644
--- a/ctypes_configure/stdoutcapture.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-A quick hack to capture stdout/stderr.
-"""
-
-import os, sys
-
-
-class Capture:
-    
-    def __init__(self, mixed_out_err = False):
-        "Start capture of the Unix-level stdout and stderr."
-        if (not hasattr(os, 'tmpfile') or
-            not hasattr(os, 'dup') or
-            not hasattr(os, 'dup2') or
-            not hasattr(os, 'fdopen')):
-            self.dummy = 1
-        else:
-            try:
-                self.tmpout = os.tmpfile()
-                if mixed_out_err:
-                    self.tmperr = self.tmpout
-                else:
-                    self.tmperr = os.tmpfile()
-            except OSError:     # bah?  on at least one Windows box
-                self.dummy = 1
-                return
-            self.dummy = 0
-            # make new stdout/stderr files if needed
-            self.localoutfd = os.dup(1)
-            self.localerrfd = os.dup(2)
-            if hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == 1:
-                self.saved_stdout = sys.stdout
-                sys.stdout = os.fdopen(self.localoutfd, 'w', 1)
-            else:
-                self.saved_stdout = None
-            if hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == 2:
-                self.saved_stderr = sys.stderr
-                sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
-            else:
-                self.saved_stderr = None
-            os.dup2(self.tmpout.fileno(), 1)
-            os.dup2(self.tmperr.fileno(), 2)
-
-    def done(self):
-        "End capture and return the captured text (stdoutfile, stderrfile)."
-        if self.dummy:
-            import cStringIO
-            return cStringIO.StringIO(), cStringIO.StringIO()
-        else:
-            os.dup2(self.localoutfd, 1)
-            os.dup2(self.localerrfd, 2)
-            if self.saved_stdout is not None:
-                f = sys.stdout
-                sys.stdout = self.saved_stdout
-                f.close()
-            else:
-                os.close(self.localoutfd)
-            if self.saved_stderr is not None:
-                f = sys.stderr
-                sys.stderr = self.saved_stderr
-                f.close()
-            else:
-                os.close(self.localerrfd)
-            self.tmpout.seek(0)
-            self.tmperr.seek(0)
-            return self.tmpout, self.tmperr
-
-
-if __name__ == '__main__':
-    # test
-    c = Capture()
-    try:
-        os.system('echo hello')
-    finally:
-        fout, ferr = c.done()
-    print 'Output:', `fout.read()`
-    print 'Error:', `ferr.read()`
diff --git a/ctypes_configure/test/__init__.py b/ctypes_configure/test/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/test/test_configure.py b/ctypes_configure/test/test_configure.py
deleted file mode 100644
--- a/ctypes_configure/test/test_configure.py
+++ /dev/null
@@ -1,212 +0,0 @@
-import py, sys, struct
-from ctypes_configure import configure
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import ctypes
-
-def test_dirent():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_reclen", ctypes.c_ushort)])
-    assert issubclass(dirent, ctypes.Structure)
-    ssize = (ctypes.sizeof(ctypes.c_long) +
-             ctypes.sizeof(ctypes.c_int) +
-             ctypes.sizeof(ctypes.c_ushort) +
-             32)
-    extra_padding = (-ssize) % ctypes.alignment(ctypes.c_long)
-
-    assert dirent._fields_ == [('_alignment', ctypes.c_long),
-                               ('_pad0', ctypes.c_char),
-                               ('_pad1', ctypes.c_char),
-                               ('_pad2', ctypes.c_char),
-                               ('_pad3', ctypes.c_char),
-                               ('d_reclen', ctypes.c_ushort),
-                               ] + [
-                               ('_pad%d' % n, ctypes.c_char)
-                                    for n in range(4, 4+32+extra_padding)]
-    assert ctypes.sizeof(dirent) == ssize + extra_padding
-    assert ctypes.alignment(dirent) == ctypes.alignment(ctypes.c_long)
-
-def test_fit_type():
-    S = configure.getstruct("struct S",
-                                  """
-           struct S {
-               signed char c;
-               unsigned char uc;
-               short s;
-               unsigned short us;
-               int i;
-               unsigned int ui;
-               long l;
-               unsigned long ul;
-               long long ll;
-               unsigned long long ull;
-               float f;
-               double d;
-           };
-                                  """,
-                                  [("c",   ctypes.c_int),
-                                   ("uc",  ctypes.c_int),
-                                   ("s",   ctypes.c_uint),
-                                   ("us",  ctypes.c_int),
-                                   ("i",   ctypes.c_int),
-                                   ("ui",  ctypes.c_int),
-                                   ("l",   ctypes.c_int),
-                                   ("ul",  ctypes.c_int),
-                                   ("ll",  ctypes.c_int),
-                                   ("ull", ctypes.c_int),
-                                   ("f",   ctypes.c_double),
-                                   ("d",   ctypes.c_float)])
-    assert issubclass(S, ctypes.Structure)
-    fields = dict(S._fields_)
-    assert fields["c"] == ctypes.c_byte
-    assert fields["uc"] == ctypes.c_ubyte
-    assert fields["s"] == ctypes.c_short
-    assert fields["us"] == ctypes.c_ushort
-    assert fields["i"] == ctypes.c_int
-    assert fields["ui"] == ctypes.c_uint
-    assert fields["l"] == ctypes.c_long
-    assert fields["ul"] == ctypes.c_ulong
-    assert fields["ll"] == ctypes.c_longlong
-    assert fields["ull"] == ctypes.c_ulonglong
-    assert fields["f"] == ctypes.c_float
-    assert fields["d"] == ctypes.c_double
-
-def test_simple_type():
-    ctype = configure.getsimpletype('test_t',
-                                          'typedef unsigned short test_t;',
-                                          ctypes.c_int)
-    assert ctype == ctypes.c_ushort
-
-def test_constant_integer():
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (6*7)')
-    assert value == 42
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (-2147483648LL)')
-    assert value == -2147483648
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (3333333333ULL)')
-    assert value == 3333333333
-
-def test_defined():
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE', '')
-    assert not res
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE',
-                                     '#define ALFKJLKJFLKJFKLEJDLKEWMECEE')
-    assert res
-
-def test_configure():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform.h')
-    test_h.write('#define XYZZY 42\n')
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-
-    res = configure.configure(CConfig)
-    assert issubclass(res['FILE'], ctypes.Structure)
-    assert res == {'FILE': res['FILE'],
-                   'ushort': ctypes.c_ushort,
-                   'XYZZY': 42}
-
-def test_ifdef():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines = ['/* a C comment */',
-                                  '#define XYZZY 42',
-                                  'typedef int foo;',
-                                  'struct s {',
-                                  'int i;',
-                                  'double f;'
-                                  '};'])
-
-
-        s = configure.Struct('struct s', [('i', ctypes.c_int)],
-                                   ifdef='XYZZY')
-        z = configure.Struct('struct z', [('i', ctypes.c_int)],
-                                   ifdef='FOOBAR')
-
-        foo = configure.SimpleType('foo', ifdef='XYZZY')
-        bar = configure.SimpleType('bar', ifdef='FOOBAR')
-
-    res = configure.configure(CConfig)
-    assert res['s'] is not None
-    assert res['z'] is None
-    assert res['foo'] is not None
-    assert res['bar'] is None
-
-def test_nested_structs():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines="""
-            struct x {
-            int foo;
-            unsigned long bar;
-            };
-            struct y {
-            char c;
-            struct x x;
-            };
-            """.split("\n"))
-
-        x = configure.Struct("struct x", [("bar", ctypes.c_short)])
-        y = configure.Struct("struct y", [("x", x)])
-
-    res = configure.configure(CConfig)
-    c_x = res["x"]
-    c_y = res["y"]
-    c_y_fields = dict(c_y._fields_)
-    assert issubclass(c_x , ctypes.Structure)
-    assert issubclass(c_y, ctypes.Structure)
-    assert c_y_fields["x"] is c_x
-
-def test_array():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_name", ctypes.c_char * 0)])
-    assert dirent.d_name.size == 32
-
-def test_has():
-    assert configure.has("x", "int x = 3;")
-    assert not configure.has("x", "")
-    # has() should also not crash if it is given an invalid #include
-    assert not configure.has("x", "#include <some/path/which/cannot/exist>")
-
-def test_check_eci():
-    eci = ExternalCompilationInfo()
-    assert configure.check_eci(eci)
-    eci = ExternalCompilationInfo(libraries=['some_name_that_doesnt_exist_'])
-    assert not configure.check_eci(eci)
-
-def test_sizeof():
-    assert configure.sizeof("char", ExternalCompilationInfo()) == 1
-
-def test_memory_alignment():
-    a = configure.memory_alignment()
-    print a
-    assert a % struct.calcsize("P") == 0
diff --git a/ctypes_configure/test/test_dumpcache.py b/ctypes_configure/test/test_dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/test/test_dumpcache.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import ctypes
-from ctypes_configure import configure, dumpcache
-from ctypes_configure.cbuild import ExternalCompilationInfo
-
-
-def test_cache():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform2.h')
-    test_h.write('#define XYZZY 42\n'
-                 "#define large 2147483648L\n")
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform2.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-        XUZ = configure.Has('XUZ')
-        large = configure.DefinedConstantInteger('large')
-        undef = configure.Defined('really_undefined')
-
-    res = configure.configure(CConfig)
-
-    cachefile = configdir.join('cache')
-    dumpcache.dumpcache('', str(cachefile), res)
-
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['XYZZY'] == res['XYZZY']
-    assert d['ushort'] == res['ushort']
-    assert d['FILE']._fields_ == res['FILE']._fields_
-    assert d['FILE'].__mro__[1:] == res['FILE'].__mro__[1:]
-    assert d['undef'] == res['undef']
-    assert d['large'] == res['large']
-    assert d['XUZ'] == res['XUZ']
-
-
-def test_cache_array():
-    configdir = configure.configdir
-    res = {'foo': ctypes.c_short * 27}
-    cachefile = configdir.join('cache_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
-
-def test_cache_array_array():
-    configdir = configure.configdir
-    res = {'foo': (ctypes.c_int * 2) * 3}
-    cachefile = configdir.join('cache_array_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -188,7 +188,7 @@
         are considered private and will not be called by
         SimpleXMLRPCServer.
 
-        If a registered function matches a XML-RPC request, then it
+        If a registered function matches an XML-RPC request, then it
         will be called instead of the registered instance.
 
         If the optional allow_dotted_names argument is true and the
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -274,7 +274,7 @@
     Even though IOBase does not declare read, readinto, or write because
     their signatures will vary, implementations and clients should
     consider those methods part of the interface. Also, implementations
-    may raise a IOError when operations they do not support are called.
+    may raise an IOError when operations they do not support are called.
 
     The basic type used for binary data read from or written to a file is
     the bytes type. Method arguments may also be bytearray or memoryview of
diff --git a/lib-python/2.7/calendar.py b/lib-python/2.7/calendar.py
--- a/lib-python/2.7/calendar.py
+++ b/lib-python/2.7/calendar.py
@@ -174,22 +174,23 @@
         Like itermonthdates(), but will yield (day number, weekday number)
         tuples. For days outside the specified month the day number is 0.
         """
-        for date in self.itermonthdates(year, month):
-            if date.month != month:
-                yield (0, date.weekday())
-            else:
-                yield (date.day, date.weekday())
+        for i, d in enumerate(self.itermonthdays(year, month), self.firstweekday):
+            yield d, i % 7
 
     def itermonthdays(self, year, month):
         """
         Like itermonthdates(), but will yield day numbers. For days outside
         the specified month the day number is 0.
         """
-        for date in self.itermonthdates(year, month):
-            if date.month != month:
-                yield 0
-            else:
-                yield date.day
+        day1, ndays = monthrange(year, month)
+        days_before = (day1 - self.firstweekday) % 7
+        for _ in range(days_before):
+            yield 0
+        for d in range(1, ndays + 1):
+            yield d
+        days_after = (self.firstweekday - day1 - ndays) % 7
+        for _ in range(days_after):
+            yield 0
 
     def monthdatescalendar(self, year, month):
         """
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -21,7 +21,7 @@
 usage of the Chunk class defined here is to instantiate an instance at
 the start of each chunk and read from the instance until it reaches
 the end, after which a new instance can be instantiated.  At the end
-of the file, creating a new instance will fail with a EOFError
+of the file, creating a new instance will fail with an EOFError
 exception.
 
 Usage:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -252,7 +252,7 @@
     """
     def __init__(self, errors='strict'):
         """
-        Creates a IncrementalDecoder instance.
+        Creates an IncrementalDecoder instance.
 
         The IncrementalDecoder may use different error handling schemes by
         providing the errors keyword argument. See the module docstring
@@ -1012,7 +1012,7 @@
     """
     Encoding iterator.
 
-    Encodes the input strings from the iterator using a IncrementalEncoder.
+    Encodes the input strings from the iterator using an IncrementalEncoder.
 
     errors and kwargs are passed through to the IncrementalEncoder
     constructor.
@@ -1030,7 +1030,7 @@
     """
     Decoding iterator.
 
-    Decodes the input strings from the iterator using a IncrementalDecoder.
+    Decodes the input strings from the iterator using an IncrementalDecoder.
 
     errors and kwargs are passed through to the IncrementalDecoder
     constructor.
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -33,6 +33,10 @@
     from __pypy__ import reversed_dict as _reversed_dict
 except ImportError:
     _reversed_dict = None     # don't have ordered dicts
+try:
+    from __pypy__ import dict_popitem_first as _dict_popitem_first
+except ImportError:
+    _dict_popitem_first = None
 
 try:
     from thread import get_ident as _get_ident
@@ -44,6 +48,17 @@
 ### OrderedDict
 ################################################################################
 
+if _dict_popitem_first is None:
+    def _dict_popitem_first(self):
+        it = dict.iteritems(self)
+        try:
+            k, v = it.next()
+        except StopIteration:
+            raise KeyError('dictionary is empty')
+        dict.__delitem__(self, k)
+        return (k, v)
+
+
 class OrderedDict(dict):
     '''Dictionary that remembers insertion order.
 
@@ -68,12 +83,7 @@
         if last:
             return dict.popitem(self)
         else:
-            it = dict.__iter__(self)
-            try:
-                k = it.next()
-            except StopIteration:
-                raise KeyError('dictionary is empty')
-            return (k, self.pop(k))
+            return _dict_popitem_first(self)
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -113,7 +113,7 @@
     """
     if t is None: t = time.time()
     year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7]
-    return "%s %02d-%s-%04d %02d:%02d:%02d GMT" % (
+    return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
         DAYS[wday], mday, MONTHS[mon-1], year, hour, min, sec)
 
 
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,3 +1,4 @@
+import functools
 import unittest
 from ctypes import *
 from ctypes.test import need_symbol
@@ -248,6 +249,40 @@
         self.assertEqual(result,
                          callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5))
 
+    def test_callback_large_struct(self):
+        class Check: pass
+
+        class X(Structure):
+            _fields_ = [
+                ('first', c_ulong),
+                ('second', c_ulong),
+                ('third', c_ulong),
+            ]
+
+        def callback(check, s):
+            check.first = s.first
+            check.second = s.second
+            check.third = s.third
+
+        check = Check()
+        s = X()
+        s.first = 0xdeadbeef
+        s.second = 0xcafebabe
+        s.third = 0x0bad1dea
+
+        CALLBACK = CFUNCTYPE(None, X)
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_cbk_large_struct
+        func.argtypes = (X, CALLBACK)
+        func.restype = None
+        # the function just calls the callback with the passed structure
+        func(s, CALLBACK(functools.partial(callback, check)))
+        self.assertEqual(check.first, s.first)
+        self.assertEqual(check.second, s.second)
+        self.assertEqual(check.third, s.third)
+        self.assertEqual(check.first, 0xdeadbeef)
+        self.assertEqual(check.second, 0xcafebabe)
+        self.assertEqual(check.third, 0x0bad1dea)
 
 ################################################################
 
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -1,6 +1,7 @@
 import unittest
-import os
+import os.path
 import sys
+from test import test_support
 from ctypes import *
 from ctypes.util import find_library
 from ctypes.test import is_resource_enabled
@@ -65,28 +66,10 @@
         if self.gle:
             self.gle.gleGetJoinStyle
 
-# On platforms where the default shared library suffix is '.so',
-# at least some libraries can be loaded as attributes of the cdll
-# object, since ctypes now tries loading the lib again
-# with '.so' appended of the first try fails.
-#
-# Won't work for libc, unfortunately.  OTOH, it isn't
-# needed for libc since this is already mapped into the current
-# process (?)
-#
-# On MAC OSX, it won't work either, because dlopen() needs a full path,
-# and the default suffix is either none or '.dylib'.
- at unittest.skip('test disabled')
- at unittest.skipUnless(os.name=="posix" and sys.platform != "darwin",
-                     'test not suitable for this platform')
-class LoadLibs(unittest.TestCase):
-    def test_libm(self):
-        import math
-        libm = cdll.libm
-        sqrt = libm.sqrt
-        sqrt.argtypes = (c_double,)
-        sqrt.restype = c_double
-        self.assertEqual(sqrt(2), math.sqrt(2))
+    def test_shell_injection(self):
+        result = find_library('; echo Hello shell > ' + test_support.TESTFN)
+        self.assertFalse(os.path.lexists(test_support.TESTFN))
+        self.assertIsNone(result)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises((TypeError, ValueError),
+        self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
     def test_fom_buffer_with_offset(self):
@@ -77,5 +77,13 @@
         self.assertRaises(ValueError,
                           (c_int * 1).from_buffer_copy, a, 16 * sizeof(c_int))
 
+    def test_abstract(self):
+        self.assertRaises(TypeError, Array.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Structure.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Union.from_buffer, bytearray(10))
+        self.assertRaises(TypeError, Array.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, Structure.from_buffer_copy, b"123")
+        self.assertRaises(TypeError, Union.from_buffer_copy, b"123")
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py
--- a/lib-python/2.7/ctypes/test/test_numbers.py
+++ b/lib-python/2.7/ctypes/test/test_numbers.py
@@ -77,7 +77,7 @@
             self.assertEqual(t(v).value, truth(v))
 
     def test_typeerror(self):
-        # Only numbers are allowed in the contructor,
+        # Only numbers are allowed in the constructor,
         # otherwise TypeError is raised
         for t in signed_types + unsigned_types + float_types:
             self.assertRaises(TypeError, t, "")
diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py
--- a/lib-python/2.7/ctypes/test/test_structures.py
+++ b/lib-python/2.7/ctypes/test/test_structures.py
@@ -106,7 +106,7 @@
         self.assertEqual(alignment(XX), alignment(X))
         self.assertEqual(sizeof(XX), calcsize("3s 3s 0s"))
 
-    def test_emtpy(self):
+    def test_empty(self):
         # I had problems with these
         #
         # Although these are pathological cases: Empty Structures!
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -1,4 +1,6 @@
-import sys, os
+import os
+import subprocess
+import sys
 
 # find_library(name) returns the pathname of a library, or None.
 if os.name == "nt":
@@ -87,25 +89,28 @@
 
     def _findLib_gcc(name):
         import tempfile
+        # Run GCC's linker with the -t (aka --trace) option and examine the
+        # library name it prints out. The GCC command will fail because we
+        # haven't supplied a proper program with main(), but that does not
+        # matter.
         expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
-        fdout, ccout = tempfile.mkstemp()
-        os.close(fdout)
-        cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit 10; fi;' \
-              'LANG=C LC_ALL=C $CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name
+        cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit; fi;' \
+              'LANG=C LC_ALL=C $CC -Wl,-t -o "$2" 2>&1 -l"$1"'
+
+        temp = tempfile.NamedTemporaryFile()
         try:
-            f = os.popen(cmd)
-            try:
-                trace = f.read()
-            finally:
-                rv = f.close()
+            proc = subprocess.Popen((cmd, '_findLib_gcc', name, temp.name),
+                                    shell=True,
+                                    stdout=subprocess.PIPE)

From pypy.commits at gmail.com  Mon Feb 13 06:42:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 03:42:26 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: To make this branch more
 incrementally mergeable to trunk, add BACKCOMPAT
Message-ID: <58a19ba2.ccaddf0a.f9559.734a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90079:3b763661eb95
Date: 2017-02-13 12:11 +0100
http://bitbucket.org/pypy/pypy/changeset/3b763661eb95/

Log:	To make this branch more incrementally mergeable to trunk, add
	BACKCOMPAT comments and keep it working with both the current and
	new ways

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -77,8 +77,7 @@
 
     def getname(self, space):
         try:
-            # YYY should be text_w?
-            return space.bytes_w(space.getattr(self, space.newtext('__name__')))
+            return space.text_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return '?'
@@ -842,7 +841,7 @@
 
     def new_interned_w_str(self, w_s):
         assert isinstance(w_s, W_Root)   # and is not None
-        s = self.bytes_w(w_s)
+        s = self.text_w(w_s)
         if not we_are_translated():
             assert type(s) is str
         w_s1 = self.interned_strings.get(s)
@@ -857,7 +856,7 @@
             assert type(s) is str
         w_s1 = self.interned_strings.get(s)
         if w_s1 is None:
-            w_s1 = self.newbytes(s)
+            w_s1 = self.newtext(s)
             self.interned_strings.set(s, w_s1)
         return w_s1
 
@@ -1608,12 +1607,17 @@
         return None if self.is_none(w_obj) else self.bytes_w(w_obj)
 
     def bytes_w(self, w_obj):
+        "Takes a bytes object and returns an unwrapped RPython bytestring."
         return w_obj.str_w(self)
-    text_w = bytes_w # equivalent to identifier_w on Python3
 
-    @not_rpython
+    def text_w(self, w_obj):
+        """Takes a string object (unicode in Python 3) and returns an
+        unwrapped RPython bytestring."""
+        return w_obj.str_w(self)
+
+    #@not_rpython    BACKCOMPAT: should be replaced with bytes_w or text_w
     def str_w(self, w_obj):
-        # XXX there are still some tests that call it
+        """For tests only."""
         return self.bytes_w(w_obj)
 
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -89,9 +89,9 @@
         for typedef, cls in builtin_type_classes.items():
             w_type = self.gettypeobject(typedef)
             self.builtin_types[typedef.name] = w_type
-            if typedef.name != "str":
+            if 1: # typedef.name != "str":      BACKCOMPAT
                 setattr(self, 'w_' + typedef.name, w_type)
-            else:
+            if typedef.name == "str":
                 self.w_bytes = w_type
             self._interplevel_classes[w_type] = cls
         self.w_text = self.w_bytes # this is w_unicode on Py3
@@ -133,7 +133,11 @@
         assert typedef is not None
         return self.fromcache(TypeCache).getorbuild(typedef)
 
-    @not_rpython # only for tests
+    # BACKCOMPAT: this function is still accepted for backward
+    # compatibility, but its usage should be progressively removed
+    # everywhere apart from tests.
+    #@not_rpython # only for tests
+    @specialize.argtype(1)
     def wrap(self, x):
         """ Wraps the Python value 'x' into one of the wrapper classes. This
         should only be used for tests, in real code you need to use the
@@ -166,6 +170,12 @@
         if is_valid_int(x):
             return self.newint(x)
 
+        return self._wrap_not_rpython(x)
+
+    def _wrap_not_rpython(self, x):
+        "NOT_RPYTHON"
+        # _____ this code is here to support testing only _____
+
         # wrap() of a container works on CPython, but the code is
         # not RPython.  Don't use -- it is kept around mostly for tests.
         # Use instead newdict(), newlist(), newtuple().

From pypy.commits at gmail.com  Mon Feb 13 08:34:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 05:34:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a1b5de.92d41c0a.ba436.aaf9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90080:82e97c392624
Date: 2017-02-13 14:33 +0100
http://bitbucket.org/pypy/pypy/changeset/82e97c392624/

Log:	hg merge default

diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -33,6 +33,10 @@
     from __pypy__ import reversed_dict as _reversed_dict
 except ImportError:
     _reversed_dict = None     # don't have ordered dicts
+try:
+    from __pypy__ import dict_popitem_first as _dict_popitem_first
+except ImportError:
+    _dict_popitem_first = None
 
 try:
     from thread import get_ident as _get_ident
@@ -44,6 +48,17 @@
 ### OrderedDict
 ################################################################################
 
+if _dict_popitem_first is None:
+    def _dict_popitem_first(self):
+        it = dict.iteritems(self)
+        try:
+            k, v = it.next()
+        except StopIteration:
+            raise KeyError('dictionary is empty')
+        dict.__delitem__(self, k)
+        return (k, v)
+
+
 class OrderedDict(dict):
     '''Dictionary that remembers insertion order.
 
@@ -68,12 +83,7 @@
         if last:
             return dict.popitem(self)
         else:
-            it = dict.__iter__(self)
-            try:
-                k = it.next()
-            except StopIteration:
-                raise KeyError('dictionary is empty')
-            return (k, self.pop(k))
+            return _dict_popitem_first(self)
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -78,6 +78,7 @@
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
         'reversed_dict'             : 'interp_dict.reversed_dict',
+        'dict_popitem_first'        : 'interp_dict.dict_popitem_first',
         'delitem_if_value_is'       : 'interp_dict.delitem_if_value_is',
         'move_to_end'               : 'interp_dict.move_to_end',
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -45,6 +45,27 @@
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_reversed_dict(space)
 
+def dict_popitem_first(space, w_obj):
+    """Interp-level implementation of OrderedDict.popitem(last=False).
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_popitem_first(space)
+
+def delitem_if_value_is(space, w_obj, w_key, w_value):
+    """Atomic equivalent to: 'if dict.get(key) is value: del dict[key]'.
+
+    SPECIAL USE CASES ONLY!  Avoid using on dicts which are specialized,
+    e.g. to int or str keys, because it switches to the object strategy.
+    Also, the 'is' operation is really pointer equality, so avoid using
+    it if 'value' is an immutable object like int or str.
+    """
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if not isinstance(w_obj, W_DictMultiObject):
+        raise OperationError(space.w_TypeError, space.w_None)
+    return w_obj.nondescr_delitem_if_value_is(space, w_key, w_value)
+
 @unwrap_spec(last=bool)
 def move_to_end(space, w_obj, w_key, last=True):
     """Move the key in a dictionary object into the first or last position.
@@ -59,16 +80,3 @@
     if not isinstance(w_obj, W_DictMultiObject):
         raise OperationError(space.w_TypeError, space.w_None)
     return w_obj.nondescr_move_to_end(space, w_key, last)
-
-def delitem_if_value_is(space, w_obj, w_key, w_value):
-    """Atomic equivalent to: 'if dict.get(key) is value: del dict[key]'.
-
-    SPECIAL USE CASES ONLY!  Avoid using on dicts which are specialized,
-    e.g. to int or str keys, because it switches to the object strategy.
-    Also, the 'is' operation is really pointer equality, so avoid using
-    it if 'value' is an immutable object like int or str.
-    """
-    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
-    if not isinstance(w_obj, W_DictMultiObject):
-        raise OperationError(space.w_TypeError, space.w_None)
-    return w_obj.nondescr_delitem_if_value_is(space, w_key, w_value)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -102,13 +102,6 @@
             result[key] = val
         return result
 
-    def missing_method(w_dict, space, w_key):
-        if not space.is_w(space.type(w_dict), space.w_dict):
-            w_missing = space.lookup(w_dict, '__missing__')
-            if w_missing is not None:
-                return space.get_and_call_function(w_missing, w_dict, w_key)
-        return None
-
     def initialize_content(self, list_pairs_w):
         for w_k, w_v in list_pairs_w:
             self.setitem(w_k, w_v)
@@ -191,9 +184,11 @@
         if w_value is not None:
             return w_value
 
-        w_missing_item = self.missing_method(space, w_key)
-        if w_missing_item is not None:
-            return w_missing_item
+        # if there is a __missing__ method, call it
+        if not space.is_w(space.type(self), space.w_dict):
+            w_missing = space.lookup(self, '__missing__')
+            if w_missing is not None:
+                return space.get_and_call_function(w_missing, self, w_key)
 
         space.raise_key_error(w_key)
 
@@ -206,6 +201,14 @@
         except KeyError:
             space.raise_key_error(w_key)
 
+    def internal_delitem(self, w_key):
+        try:
+            self.delitem(w_key)
+        except KeyError:
+            raise oefmt(self.space.w_RuntimeError,
+                        "an internal 'del' on the dictionary failed to find "
+                        "the key")
+
     def descr_copy(self, space):
         """D.copy() -> a shallow copy of D"""
         return self.copy()
@@ -234,6 +237,14 @@
             w_keys = self.w_keys()
             return space.call_method(w_keys, '__reversed__')
 
+    def nondescr_delitem_if_value_is(self, space, w_key, w_value):
+        """Not exposed directly to app-level, but used by
+        _weakref._remove_dead_weakref and via __pypy__.delitem_if_value_is().
+        """
+        strategy = self.ensure_object_strategy()
+        d = strategy.unerase(self.dstorage)
+        objectmodel.delitem_if_value_is(d, w_key, w_value)
+
     def nondescr_move_to_end(self, space, w_key, last_flag):
         """Not exposed directly to app-level, but via __pypy__.move_to_end().
         """
@@ -246,7 +257,7 @@
             if w_value is None:
                 space.raise_key_error(w_key)
             else:
-                self.delitem(w_key)
+                self.internal_delitem(w_key)
                 if last_flag:
                     self.setitem(w_key, w_value)
                 else:
@@ -265,13 +276,14 @@
                     for i in range(len(keys_w)):
                         self.setitem(keys_w[i], values_w[i])
 
-    def nondescr_delitem_if_value_is(self, space, w_key, w_value):
-        """Not exposed directly to app-level, but used by
-        _weakref._remove_dead_weakref and via __pypy__.delitem_if_value_is().
+    def nondescr_popitem_first(self, space):
+        """Not exposed directly to app-level, but via __pypy__.popitem_first().
         """
-        strategy = self.ensure_object_strategy()
-        d = strategy.unerase(self.dstorage)
-        objectmodel.delitem_if_value_is(d, w_key, w_value)
+        w_key, w_value = self.iteritems().next_item()
+        if w_key is None:
+            raise oefmt(space.w_KeyError, "popitem(): dictionary is empty")
+        self.internal_delitem(w_key)
+        return space.newtuple([w_key, w_value])
 
     def descr_clear(self, space):
         """D.clear() -> None.  Remove all items from D."""
@@ -288,6 +300,13 @@
         corresponding value\nIf key is not found, d is returned if given,
         otherwise KeyError is raised
         """
+        strategy = self.get_strategy()
+        if strategy.has_pop:
+            try:
+                return strategy.pop(self, w_key, w_default)
+            except KeyError:
+                raise space.raise_key_error(w_key)
+        # fall-back
         w_item = self.getitem(w_key)
         if w_item is None:
             if w_default is not None:
@@ -295,7 +314,7 @@
             else:
                 space.raise_key_error(w_key)
         else:
-            self.delitem(w_key)
+            self.internal_delitem(w_key)
             return w_item
 
     def descr_popitem(self, space):
@@ -536,8 +555,8 @@
 
     has_iterreversed = False
     has_move_to_end = False
-    # no 'getiterreversed' and no 'move_to_end': no default
-    # implementation available
+    has_pop = False
+    # ^^^ no default implementation available for these methods
 
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
         iteritems = self.iteritems(w_dict)
@@ -824,6 +843,9 @@
     if hasattr(dictimpl, 'move_to_end'):
         dictimpl.has_move_to_end = True
 
+    if hasattr(dictimpl, 'pop'):
+        dictimpl.has_pop = True
+
     @jit.look_inside_iff(lambda self, w_dict, w_updatedict:
                          w_dict_unrolling_heuristic(w_dict))
     def rev_update1_dict_dict(self, w_dict, w_updatedict):
@@ -977,6 +999,21 @@
         key, value = self.unerase(w_dict.dstorage).popitem()
         return (self.wrap(key), value)
 
+    def pop(self, w_dict, w_key, w_default):
+        space = self.space
+        if self.is_correct_type(w_key):
+            key = self.unwrap(w_key)
+            d = self.unerase(w_dict.dstorage)
+            if w_default is None:
+                return d.pop(key)
+            else:
+                return d.pop(key, w_default)
+        elif self._never_equal_to(space.type(w_key)):
+            raise KeyError
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.get_strategy().pop(w_dict, w_key, w_default)
+
     def clear(self, w_dict):
         self.unerase(w_dict.dstorage).clear()
 
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -266,10 +266,36 @@
         d = {1: 2, 3: 4, 5: 6}
         it = __pypy__.reversed_dict(d)
         key = next(it)
-        assert key in [1, 3, 5]
+        assert key in [1, 3, 5]   # on CPython, dicts are not ordered
         del d[key]
         raises(RuntimeError, next, it)
 
+    def test_dict_popitem_first(self):
+        import __pypy__
+        d = {"a": 5}
+        assert __pypy__.dict_popitem_first(d) == ("a", 5)
+        raises(KeyError, __pypy__.dict_popitem_first, d)
+
+        def kwdict(**k):
+            return k
+        d = kwdict(a=55)
+        assert __pypy__.dict_popitem_first(d) == ("a", 55)
+        raises(KeyError, __pypy__.dict_popitem_first, d)
+
+    def test_delitem_if_value_is(self):
+        import __pypy__
+        class X:
+            pass
+        x2 = X()
+        x3 = X()
+        d = {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {2: x2, 3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x2)
+        assert d == {3: x3}
+        __pypy__.delitem_if_value_is(d, 2, x3)
+        assert d == {3: x3}
+
     def test_move_to_end(self):
         import __pypy__
         raises(KeyError, __pypy__.move_to_end, {}, 'foo')
@@ -294,20 +320,6 @@
                     assert list(d) == [key] + other_keys
                 raises(KeyError, __pypy__.move_to_end, d, key * 3, last=last)
 
-    def test_delitem_if_value_is(self):
-        import __pypy__
-        class X:
-            pass
-        x2 = X()
-        x3 = X()
-        d = {2: x2, 3: x3}
-        __pypy__.delitem_if_value_is(d, 2, x3)
-        assert d == {2: x2, 3: x3}
-        __pypy__.delitem_if_value_is(d, 2, x2)
-        assert d == {3: x3}
-        __pypy__.delitem_if_value_is(d, 2, x3)
-        assert d == {3: x3}
-
     def test_keys(self):
         d = {1: 2, 3: 4}
         kys = list(d.keys())
@@ -618,6 +630,18 @@
             else:
                 assert False, 'Expected KeyError'
 
+    def test_pop_switching_strategy(self):
+        class Foo:
+            def __hash__(self):
+                return hash("a")
+            def __eq__(self, other):
+                return other == "a"
+        d = {"a": 42}
+        x = d.pop(Foo())
+        assert x == 42 and len(d) == 0
+        d = {"b": 43}
+        raises(KeyError, d.pop, Foo())
+
     def test_no_len_on_dict_iter(self):
         iterable = {1: 2, 3: 4}
         raises(TypeError, len, iter(iterable))
@@ -694,6 +718,31 @@
         setattr(a, s, 123)
         assert holder.seen is s
 
+    def test_internal_delitem(self):
+        class K:
+            def __hash__(self):
+                return 42
+            def __eq__(self, other):
+                if is_equal[0]:
+                    is_equal[0] -= 1
+                    return True
+                return False
+        is_equal = [0]
+        k1 = K()
+        k2 = K()
+        d = {k1: 1, k2: 2}
+        k3 = K()
+        is_equal = [1]
+        try:
+            x = d.pop(k3)
+        except RuntimeError:
+            # This used to give a Fatal RPython error: KeyError.
+            # Now at least it should raise an app-level RuntimeError,
+            # or just work.
+            assert len(d) == 2
+        else:
+            assert (x == 1 or x == 2) and len(d) == 1
+
 
 class AppTestDictViews:
     def test_dictview(self):

From pypy.commits at gmail.com  Mon Feb 13 09:23:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 06:23:59 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix some more space.wrap(),
 which still fail ztranslation tests
Message-ID: <58a1c17f.c3161c0a.8445b.b723@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90082:6d64ad55df3b
Date: 2017-02-13 15:23 +0100
http://bitbucket.org/pypy/pypy/changeset/6d64ad55df3b/

Log:	fix some more space.wrap(), which still fail ztranslation tests but
	work when really translated for now (I think it's ok)

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -422,9 +422,9 @@
 
         if num_bytes <= 0:
             if rwbuffer:
-                return space.wrap(0)
+                return space.newint(0)
             else:
-                return space.wrap("")
+                return space.newbytes("")
 
         with rffi.scoped_alloc_buffer(num_bytes) as buf:
             while True:

From pypy.commits at gmail.com  Mon Feb 13 09:23:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 06:23:57 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: Tweak the fake objspace
Message-ID: <58a1c17d.499adf0a.21fff.a3d6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90081:cb1017183c06
Date: 2017-02-13 15:21 +0100
http://bitbucket.org/pypy/pypy/changeset/cb1017183c06/

Log:	Tweak the fake objspace

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1057,6 +1057,8 @@
             assert space._code_of_sys_exc_info is None
             space._code_of_sys_exc_info = code
         #
+        if hasattr(space, '_see_interp2app'):
+            space._see_interp2app(gateway)      # only for fake/objspace.py
         return fn
 
 
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -343,6 +343,11 @@
         raise oefmt(space.w_AttributeError,
                     "generic property has no __objclass__")
 
+    def spacebind(self, space):
+        if hasattr(space, '_see_getsetproperty'):
+            space._see_getsetproperty(self)      # only for fake/objspace.py
+        return self
+
 
 def interp_attrproperty(name, cls, doc=None, wrapfn=None):
     "NOT_RPYTHON: initialization-time only"
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -123,6 +123,9 @@
                  'unicode', 'complex', 'slice', 'bool', 'basestring', 'object',
                  'bytearray', 'buffer', 'set', 'frozenset']
 
+INTERP_TYPES = ['function', 'builtin_function', 'module', 'getset_descriptor',
+                'instance', 'classobj']
+
 class FakeObjSpace(ObjSpace):
     is_fake_objspace = True
 
@@ -211,10 +214,8 @@
     @not_rpython
     def wrap(self, x):
         if not we_are_translated():
-            if isinstance(x, gateway.interp2app):
-                self._see_interp2app(x)
-            if isinstance(x, GetSetProperty):
-                self._see_getsetproperty(x)
+            if isinstance(x, W_Root):
+                x.spacebind(self)
         if isinstance(x, r_singlefloat):
             self._wrap_not_rpython(x)
         if isinstance(x, list):
@@ -225,6 +226,7 @@
 
     @not_rpython
     def _see_interp2app(self, interp2app):
+        """Called by GatewayCache.build()"""
         activation = interp2app._code.activation
         def check():
             scope_w = [w_some_obj()] * NonConstant(42)
@@ -235,6 +237,7 @@
 
     @not_rpython
     def _see_getsetproperty(self, getsetproperty):
+        """Called by GetSetProperty.spacebind()"""
         space = self
         def checkprop():
             getsetproperty.fget(getsetproperty, space, w_some_obj())
@@ -358,6 +361,7 @@
         ann = t.buildannotator()
         def _do_startup():
             self.threadlocals.enter_thread(self)
+            W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj())
         ann.build_types(_do_startup, [], complete_now=False)
         if func is not None:
             ann.build_types(func, argtypes, complete_now=False)
@@ -418,7 +422,9 @@
 @specialize.memo()
 def see_typedef(space, typedef):
     assert isinstance(typedef, TypeDef)
-    if typedef.name not in BUILTIN_TYPES:
+    if typedef.name not in BUILTIN_TYPES and typedef.name not in INTERP_TYPES:
+        print
+        print '------ seeing typedef %r ------' % (typedef.name,)
         for name, value in typedef.rawdict.items():
             space.wrap(value)
 
diff --git a/pypy/objspace/fake/test/test_checkmodule.py b/pypy/objspace/fake/test/test_checkmodule.py
--- a/pypy/objspace/fake/test/test_checkmodule.py
+++ b/pypy/objspace/fake/test/test_checkmodule.py
@@ -19,7 +19,7 @@
     space = FakeObjSpace()
     assert len(space._seen_extras) == 0
     assert len(check) == 0
-    space.wrap(interp2app(lambda space: see()))
+    interp2app(lambda space: see()).spacebind(space)
     assert len(space._seen_extras) == 1
     assert len(check) == 0
     space.translates()

From pypy.commits at gmail.com  Mon Feb 13 09:32:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 06:32:58 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: an extra fix
Message-ID: <58a1c39a.018e1c0a.e61b9.bbec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90083:b1ec7c118e2b
Date: 2017-02-13 15:32 +0100
http://bitbucket.org/pypy/pypy/changeset/b1ec7c118e2b/

Log:	an extra fix

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -58,7 +58,10 @@
 
     def bigint_w(self, space, allow_conversion=True):
         from rpython.rlib.rbigint import rbigint
-        return rbigint.fromint(NonConstant(42))
+        x = 42
+        if we_are_translated():
+            x = NonConstant(x)
+        return rbigint.fromint(x)
 
 class W_MyListObj(W_MyObject):
     def append(self, w_other):

From pypy.commits at gmail.com  Mon Feb 13 09:41:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 06:41:13 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix for test_pypy_c
Message-ID: <58a1c589.0fb81c0a.ee3e2.c122@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90084:3bba29c8e6e8
Date: 2017-02-13 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/3bba29c8e6e8/

Log:	fix for test_pypy_c

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -131,7 +131,6 @@
             guard_no_exception(descr=...)
             p95 = call_r(..., descr=<Callr . r EF=5>)     # ll_build
             guard_no_exception(descr=...)
-            guard_nonnull(p95, descr=...)
             i96 = strlen(p95)
             i97 = int_add_ovf(i71, i96)
             guard_no_overflow(descr=...)
@@ -210,10 +209,8 @@
         guard_not_invalidated(descr=...)
         p80 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i47, descr=<Callr . i EF=3>)
         guard_no_exception(descr=...)
-        guard_nonnull(p80, descr=...)
         p53 = call_r(ConstClass(fast_str_decode_ascii), p80, descr=<Callr . r EF=4>)
         guard_no_exception(descr=...)
-        guard_nonnull(p53, descr=...)
         --TICK--
         jump(..., descr=...)
         """)

From pypy.commits at gmail.com  Mon Feb 13 09:41:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 06:41:15 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix
Message-ID: <58a1c58b.8794df0a.c66a3.a6ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90085:f50a640c6743
Date: 2017-02-13 15:40 +0100
http://bitbucket.org/pypy/pypy/changeset/f50a640c6743/

Log:	fix

diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -72,6 +72,7 @@
         for name in ('int', 'long', 'str', 'unicode', 'None', 'ValueError',
                 'OverflowError'):
             setattr(self, 'w_' + name, eval(name))
+        self.w_bytes = bytes
         import __builtin__ as __builtin__
         self.builtin = __builtin__
 

From pypy.commits at gmail.com  Mon Feb 13 09:54:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 06:54:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge py3.5
Message-ID: <58a1c8bc.0f96df0a.648cf.b843@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90086:412f0bc1a605
Date: 2017-02-13 15:53 +0100
http://bitbucket.org/pypy/pypy/changeset/412f0bc1a605/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 140082 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
diff --git a/ctypes_configure/__init__.py b/ctypes_configure/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/cbuild.py b/ctypes_configure/cbuild.py
deleted file mode 100644
--- a/ctypes_configure/cbuild.py
+++ /dev/null
@@ -1,456 +0,0 @@
-
-import os, sys, inspect, re, imp, py
-from ctypes_configure import stdoutcapture
-import distutils
-
-debug = 0
-
-configdir = py.path.local.make_numbered_dir(prefix='ctypes_configure-')
-
-class ExternalCompilationInfo(object):
-
-    _ATTRIBUTES = ['pre_include_lines', 'includes', 'include_dirs',
-                   'post_include_lines', 'libraries', 'library_dirs',
-                   'separate_module_sources', 'separate_module_files']
-    _AVOID_DUPLICATES = ['separate_module_files', 'libraries', 'includes',
-                         'include_dirs', 'library_dirs', 'separate_module_sources']
-
-    def __init__(self,
-                 pre_include_lines       = [],
-                 includes                = [],
-                 include_dirs            = [],
-                 post_include_lines      = [],
-                 libraries               = [],
-                 library_dirs            = [],
-                 separate_module_sources = [],
-                 separate_module_files   = []):
-        """
-        pre_include_lines: list of lines that should be put at the top
-        of the generated .c files, before any #include.  They shouldn't
-        contain an #include themselves.
-
-        includes: list of .h file names to be #include'd from the
-        generated .c files.
-
-        include_dirs: list of dir names that is passed to the C compiler
-
-        post_include_lines: list of lines that should be put at the top
-        of the generated .c files, after the #includes.
-
-        libraries: list of library names that is passed to the linker
-
-        library_dirs: list of dir names that is passed to the linker
-
-        separate_module_sources: list of multiline strings that are
-        each written to a .c file and compiled separately and linked
-        later on.  (If function prototypes are needed for other .c files
-        to access this, they can be put in post_include_lines.)
-
-        separate_module_files: list of .c file names that are compiled
-        separately and linked later on.  (If an .h file is needed for
-        other .c files to access this, it can be put in includes.)
-        """
-        for name in self._ATTRIBUTES:
-            value = locals()[name]
-            assert isinstance(value, (list, tuple))
-            setattr(self, name, tuple(value))
-
-    def _value(self):
-        return tuple([getattr(self, x) for x in self._ATTRIBUTES])
-
-    def __hash__(self):
-        return hash(self._value())
-
-    def __eq__(self, other):
-        return self.__class__ is other.__class__ and \
-               self._value() == other._value()
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __repr__(self):
-        info = []
-        for attr in self._ATTRIBUTES:
-            val = getattr(self, attr)
-            info.append("%s=%s" % (attr, repr(val)))
-        return "<ExternalCompilationInfo (%s)>" % ", ".join(info)
-
-    def merge(self, *others):
-        others = list(others)
-        attrs = {}
-        for name in self._ATTRIBUTES:
-            if name not in self._AVOID_DUPLICATES:
-                s = []
-                for i in [self] + others:
-                    s += getattr(i, name)
-                attrs[name] = s
-            else:
-                s = set()
-                attr = []
-                for one in [self] + others:
-                    for elem in getattr(one, name):
-                        if elem not in s:
-                            s.add(elem)
-                            attr.append(elem)
-                attrs[name] = attr
-        return ExternalCompilationInfo(**attrs)
-
-    def write_c_header(self, fileobj):
-        for line in self.pre_include_lines:
-            print >> fileobj, line
-        for path in self.includes:
-            print >> fileobj, '#include <%s>' % (path,)
-        for line in self.post_include_lines:
-            print >> fileobj, line
-
-    def _copy_attributes(self):
-        d = {}
-        for attr in self._ATTRIBUTES:
-            d[attr] = getattr(self, attr)
-        return d
-
-    def convert_sources_to_files(self, cache_dir=None, being_main=False):
-        if not self.separate_module_sources:
-            return self
-        if cache_dir is None:
-            cache_dir = configdir.join('module_cache').ensure(dir=1)
-        num = 0
-        files = []
-        for source in self.separate_module_sources:
-            while 1:
-                filename = cache_dir.join('module_%d.c' % num)
-                num += 1
-                if not filename.check():
-                    break
-            f = filename.open("w")
-            if being_main:
-                f.write("#define PYPY_NOT_MAIN_FILE\n")
-            self.write_c_header(f)
-            source = str(source)
-            f.write(source)
-            if not source.endswith('\n'):
-                f.write('\n')
-            f.close()
-            files.append(str(filename))
-        d = self._copy_attributes()
-        d['separate_module_sources'] = ()
-        d['separate_module_files'] += tuple(files)
-        return ExternalCompilationInfo(**d)
-
-    def compile_shared_lib(self):
-        self = self.convert_sources_to_files()
-        if not self.separate_module_files:
-            return self
-        lib = compile_c_module([], 'externmod', self)
-        d = self._copy_attributes()
-        d['libraries'] += (lib,)
-        d['separate_module_files'] = ()
-        d['separate_module_sources'] = ()
-        return ExternalCompilationInfo(**d)
-
-if sys.platform == 'win32':
-    so_ext = '.dll'
-else:
-    so_ext = '.so'
-
-def compiler_command():
-    # e.g. for tcc, you might set this to
-    #    "tcc -shared -o %s.so %s.c"
-    return os.getenv('PYPY_CC')
-
-def enable_fast_compilation():
-    if sys.platform == 'win32':
-        dash = '/'
-    else:
-        dash = '-'
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt:
-        opt = re.sub('%sO\d+' % dash, '%sO0' % dash, opt)
-    else:
-        opt = '%sO0' % dash
-    gcv['OPT'] = opt
-
-def ensure_correct_math():
-    if sys.platform != 'win32':
-        return # so far
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt and '/Op' not in opt:
-        opt += '/Op'
-    gcv['OPT'] = opt
-
-
-def try_compile(c_files, eci):
-    try:
-        build_executable(c_files, eci)
-        result = True
-    except (distutils.errors.CompileError,
-            distutils.errors.LinkError):
-        result = False
-    return result
-
-def compile_c_module(cfiles, modbasename, eci, tmpdir=None):
-    #try:
-    #    from distutils.log import set_threshold
-    #    set_threshold(10000)
-    #except ImportError:
-    #    print "ERROR IMPORTING"
-    #    pass
-    cfiles = [py.path.local(f) for f in cfiles]
-    if tmpdir is None:
-        tmpdir = configdir.join("module_cache").ensure(dir=1)
-    num = 0
-    cfiles += eci.separate_module_files
-    include_dirs = list(eci.include_dirs)
-    library_dirs = list(eci.library_dirs)
-    if (sys.platform == 'darwin' or    # support Fink & Darwinports
-            sys.platform.startswith('freebsd')):
-        for s in ('/sw/', '/opt/local/', '/usr/local/'):
-            if s + 'include' not in include_dirs and \
-               os.path.exists(s + 'include'):
-                include_dirs.append(s + 'include')
-            if s + 'lib' not in library_dirs and \
-               os.path.exists(s + 'lib'):
-                library_dirs.append(s + 'lib')
-
-    num = 0
-    modname = modbasename
-    while 1:
-        if not tmpdir.join(modname + so_ext).check():
-            break
-        num += 1
-        modname = '%s_%d' % (modbasename, num)
-
-    lastdir = tmpdir.chdir()
-    libraries = eci.libraries
-    ensure_correct_math()
-    try:
-        if debug: print "modname", modname
-        c = stdoutcapture.Capture(mixed_out_err = True)
-        try:
-            try:
-                if compiler_command():
-                    # GCC-ish options only
-                    from distutils import sysconfig
-                    gcv = sysconfig.get_config_vars()
-                    cmd = compiler_command().replace('%s',
-                                                     str(tmpdir.join(modname)))
-                    for dir in [gcv['INCLUDEPY']] + list(include_dirs):
-                        cmd += ' -I%s' % dir
-                    for dir in library_dirs:
-                        cmd += ' -L%s' % dir
-                    os.system(cmd)
-                else:
-                    from distutils.dist import Distribution
-                    from distutils.extension import Extension
-                    from distutils.ccompiler import get_default_compiler
-                    saved_environ = os.environ.items()
-                    try:
-                        # distutils.core.setup() is really meant for end-user
-                        # interactive usage, because it eats most exceptions and
-                        # turn them into SystemExits.  Instead, we directly
-                        # instantiate a Distribution, which also allows us to
-                        # ignore unwanted features like config files.
-                        extra_compile_args = []
-                        # ensure correct math on windows
-                        if sys.platform == 'win32':
-                            extra_compile_args.append('/Op') # get extra precision
-                        if get_default_compiler() == 'unix':
-                            old_version = False
-                            try:
-                                g = os.popen('gcc --version', 'r')
-                                verinfo = g.read()
-                                g.close()
-                            except (OSError, IOError):
-                                pass
-                            else:
-                                old_version = verinfo.startswith('2')
-                            if not old_version:
-                                extra_compile_args.extend(["-Wno-unused-label",
-                                                        "-Wno-unused-variable"])
-                        attrs = {
-                            'name': "testmodule",
-                            'ext_modules': [
-                                Extension(modname, [str(cfile) for cfile in cfiles],
-                                    include_dirs=include_dirs,
-                                    library_dirs=library_dirs,
-                                    extra_compile_args=extra_compile_args,
-                                    libraries=list(libraries),)
-                                ],
-                            'script_name': 'setup.py',
-                            'script_args': ['-q', 'build_ext', '--inplace', '--force'],
-                            }
-                        dist = Distribution(attrs)
-                        if not dist.parse_command_line():
-                            raise ValueError, "distutils cmdline parse error"
-                        dist.run_commands()
-                    finally:
-                        for key, value in saved_environ:
-                            if os.environ.get(key) != value:
-                                os.environ[key] = value
-            finally:
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = open("%s.errors" % modname, "w")
-                    fdump.write(data)
-                    fdump.close()
-            # XXX do we need to do some check on fout/ferr?
-            # XXX not a nice way to import a module
-        except:
-            print >>sys.stderr, data
-            raise
-    finally:
-        lastdir.chdir()
-    return str(tmpdir.join(modname) + so_ext)
-
-def make_module_from_c(cfile, eci):
-    cfile = py.path.local(cfile)
-    modname = cfile.purebasename
-    compile_c_module([cfile], modname, eci)
-    return import_module_from_directory(cfile.dirpath(), modname)
-
-def import_module_from_directory(dir, modname):
-    file, pathname, description = imp.find_module(modname, [str(dir)])
-    try:
-        mod = imp.load_module(modname, file, pathname, description)
-    finally:
-        if file:
-            file.close()
-    return mod
-
-
-def log_spawned_cmd(spawn):
-    def spawn_and_log(cmd, *args, **kwds):
-        if debug:
-            print ' '.join(cmd)
-        return spawn(cmd, *args, **kwds)
-    return spawn_and_log
-
-
-class ProfOpt(object):
-    #XXX assuming gcc style flags for now
-    name = "profopt"
-    
-    def __init__(self, compiler):
-        self.compiler = compiler
-
-    def first(self):
-        self.build('-fprofile-generate')
-
-    def probe(self, exe, args):
-        # 'args' is a single string typically containing spaces
-        # and quotes, which represents several arguments.
-        os.system("'%s' %s" % (exe, args))
-
-    def after(self):
-        self.build('-fprofile-use')
-
-    def build(self, option):
-        compiler = self.compiler
-        compiler.compile_extra.append(option)
-        compiler.link_extra.append(option)
-        try:
-            compiler._build()
-        finally:
-            compiler.compile_extra.pop()
-            compiler.link_extra.pop()
-            
-class CCompiler:
-
-    def __init__(self, cfilenames, eci, outputfilename=None,
-                 compiler_exe=None, profbased=None):
-        self.cfilenames = cfilenames
-        ext = ''
-        self.compile_extra = []
-        self.link_extra = []
-        self.libraries = list(eci.libraries)
-        self.include_dirs = list(eci.include_dirs)
-        self.library_dirs = list(eci.library_dirs)
-        self.compiler_exe = compiler_exe
-        self.profbased = profbased
-        if not sys.platform in ('win32', 'darwin', 'cygwin'): # xxx
-            if 'm' not in self.libraries:
-                self.libraries.append('m')
-            if 'pthread' not in self.libraries:
-                self.libraries.append('pthread')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer', '-pthread']
-            self.link_extra += ['-pthread']
-        if sys.platform == 'win32':
-            self.link_extra += ['/DEBUG'] # generate .pdb file
-        if (sys.platform == 'darwin' or    # support Fink & Darwinports
-                sys.platform.startswith('freebsd')):
-            for s in ('/sw/', '/opt/local/', '/usr/local/'):
-                if s + 'include' not in self.include_dirs and \
-                   os.path.exists(s + 'include'):
-                    self.include_dirs.append(s + 'include')
-                if s + 'lib' not in self.library_dirs and \
-                   os.path.exists(s + 'lib'):
-                    self.library_dirs.append(s + 'lib')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer']
-
-        if outputfilename is None:
-            self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext)
-        else: 
-            self.outputfilename = py.path.local(outputfilename)
-
-    def build(self, noerr=False):
-        basename = self.outputfilename.new(ext='')
-        data = ''
-        try:
-            saved_environ = os.environ.copy()
-            c = stdoutcapture.Capture(mixed_out_err = True)
-            try:
-                self._build()
-            finally:
-                # workaround for a distutils bugs where some env vars can
-                # become longer and longer every time it is used
-                for key, value in saved_environ.items():
-                    if os.environ.get(key) != value:
-                        os.environ[key] = value
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = basename.new(ext='errors').open("w")
-                    fdump.write(data)
-                    fdump.close()
-        except:
-            if not noerr:
-                print >>sys.stderr, data
-            raise
- 
-    def _build(self):
-        from distutils.ccompiler import new_compiler 
-        compiler = new_compiler(force=1)
-        if self.compiler_exe is not None:
-            for c in '''compiler compiler_so compiler_cxx
-                        linker_exe linker_so'''.split():
-                compiler.executables[c][0] = self.compiler_exe
-        compiler.spawn = log_spawned_cmd(compiler.spawn)
-        objects = []
-        for cfile in self.cfilenames: 
-            cfile = py.path.local(cfile)
-            old = cfile.dirpath().chdir() 
-            try: 
-                res = compiler.compile([cfile.basename], 
-                                       include_dirs=self.include_dirs,
-                                       extra_preargs=self.compile_extra)
-                assert len(res) == 1
-                cobjfile = py.path.local(res[0]) 
-                assert cobjfile.check()
-                objects.append(str(cobjfile))
-            finally: 
-                old.chdir() 
-        compiler.link_executable(objects, str(self.outputfilename),
-                                 libraries=self.libraries,
-                                 extra_preargs=self.link_extra,
-                                 library_dirs=self.library_dirs)
-
-def build_executable(*args, **kwds):
-    noerr = kwds.pop('noerr', False)
-    compiler = CCompiler(*args, **kwds)
-    compiler.build(noerr=noerr)
-    return str(compiler.outputfilename)
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
deleted file mode 100755
--- a/ctypes_configure/configure.py
+++ /dev/null
@@ -1,621 +0,0 @@
-#! /usr/bin/env python
-
-import os, py, sys
-import ctypes
-from ctypes_configure.cbuild import build_executable, configdir, try_compile
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import distutils
-
-# ____________________________________________________________
-#
-# Helpers for simple cases
-
-def eci_from_header(c_header_source):
-    return ExternalCompilationInfo(
-        pre_include_lines=c_header_source.split("\n")
-    )
-
-
-def getstruct(name, c_header_source, interesting_fields):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        STRUCT = Struct(name, interesting_fields)
-    return configure(CConfig)['STRUCT']
-
-def getsimpletype(name, c_header_source, ctype_hint=ctypes.c_int):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        TYPE = SimpleType(name, ctype_hint)
-    return configure(CConfig)['TYPE']
-
-def getconstantinteger(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        CONST = ConstantInteger(name)
-    return configure(CConfig)['CONST']
-
-def getdefined(macro, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        DEFINED = Defined(macro)
-    return configure(CConfig)['DEFINED']
-
-def has(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        HAS = Has(name)
-    return configure(CConfig)['HAS']
-
-def check_eci(eci):
-    """Check if a given ExternalCompilationInfo compiles and links."""
-    class CConfig:
-        _compilation_info_ = eci
-        WORKS = Works()
-    return configure(CConfig)['WORKS']
-
-def sizeof(name, eci, **kwds):
-    class CConfig:
-        _compilation_info_ = eci
-        SIZE = SizeOf(name)
-    for k, v in kwds.items():
-        setattr(CConfig, k, v)
-    return configure(CConfig)['SIZE']
-
-def memory_alignment():
-    """Return the alignment (in bytes) of memory allocations.
-    This is enough to make sure a structure with pointers and 'double'
-    fields is properly aligned."""
-    global _memory_alignment
-    if _memory_alignment is None:
-        S = getstruct('struct memory_alignment_test', """
-           struct memory_alignment_test {
-               double d;
-               void* p;
-           };
-        """, [])
-        result = ctypes.alignment(S)
-        assert result & (result-1) == 0, "not a power of two??"
-        _memory_alignment = result
-    return _memory_alignment
-_memory_alignment = None
-
-# ____________________________________________________________
-#
-# General interface
-
-class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
-        self.result = {}
-        self.info = info
-        self.entries = entries
-        
-    def get_entry_result(self, entry):
-        try:
-            return self.result[entry]
-        except KeyError:
-            pass
-        name = self.entries[entry]
-        info = self.info[name]
-        self.result[entry] = entry.build_result(info, self)
-
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
-
-
-class _CWriter(object):
-    """ A simple class which aggregates config parts
-    """
-    def __init__(self, CConfig):
-        self.path = uniquefilepath()
-        self.f = self.path.open("w")
-        self.config = CConfig
-
-    def write_header(self):
-        f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
-        print >> f, C_HEADER
-        print >> f
-
-    def write_entry(self, key, entry):
-        f = self.f
-        print >> f, 'void dump_section_%s(void) {' % (key,)
-        for line in entry.prepare_code():
-            if line and line[0] != '#':
-                line = '\t' + line
-            print >> f, line
-        print >> f, '}'
-        print >> f
-
-    def write_entry_main(self, key):
-        print >> self.f, '\tprintf("-+- %s\\n");' % (key,)
-        print >> self.f, '\tdump_section_%s();' % (key,)
-        print >> self.f, '\tprintf("---\\n");'
-
-    def start_main(self):
-        print >> self.f, 'int main(int argc, char *argv[]) {'
-
-    def close(self):
-        f = self.f
-        print >> f, '\treturn 0;'
-        print >> f, '}'
-        f.close()
-
-    def ask_gcc(self, question):
-        self.start_main()
-        self.f.write(question + "\n")
-        self.close()
-        eci = self.config._compilation_info_
-        return try_compile([self.path], eci)
-
-        
-def configure(CConfig, noerr=False):
-    """Examine the local system by running the C compiler.
-    The CConfig class contains CConfigEntry attribues that describe
-    what should be inspected; configure() returns a dict mapping
-    names to the results.
-    """
-    for attr in ['_includes_', '_libraries_', '_sources_', '_library_dirs_',
-                 '_include_dirs_', '_header_']:
-        assert not hasattr(CConfig, attr), "Found legacy attribut %s on CConfig" % (attr,)
-    entries = []
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigEntry):
-            entries.append((key, value))
-
-    if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        f = writer.f
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci, noerr=noerr))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(CConfig, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
-    else:
-        res = {}
-
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
-            writer.write_header()
-            res[key] = value.question(writer.ask_gcc)
-    return res
-
-# ____________________________________________________________
-
-
-class CConfigEntry(object):
-    "Abstract base class."
-
-class Struct(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined structure.
-    """
-    def __init__(self, name, interesting_fields, ifdef=None):
-        self.name = name
-        self.interesting_fields = interesting_fields
-        self.ifdef = ifdef
-
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield 'typedef struct {'
-        yield '    char c;'
-        yield '    ctypesplatcheck_t s;'
-        yield '} ctypesplatcheck2_t;'
-        yield ''
-        yield 'ctypesplatcheck_t s;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("align", offsetof(ctypesplatcheck2_t, s));'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        for fieldname, fieldtype in self.interesting_fields:
-            yield 'dump("fldofs %s", offsetof(ctypesplatcheck_t, %s));'%(
-                fieldname, fieldname)
-            yield 'dump("fldsize %s",   sizeof(s.%s));' % (
-                fieldname, fieldname)
-            if fieldtype in integer_class:
-                yield 's.%s = 0; s.%s = ~s.%s;' % (fieldname,
-                                                   fieldname,
-                                                   fieldname)
-                yield 'dump("fldunsigned %s", s.%s > 0);' % (fieldname,
-                                                             fieldname)
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None:
-            if not info['defined']:
-                return None
-        alignment = 1
-        layout = [None] * info['size']
-        for fieldname, fieldtype in self.interesting_fields:
-            if isinstance(fieldtype, Struct):
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                c_fieldtype = config_result.get_entry_result(fieldtype)
-                layout_addfield(layout, offset, c_fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(c_fieldtype))
-            else:
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                sign   = info.get('fldunsigned ' + fieldname, False)
-                if (size, sign) != size_and_sign(fieldtype):
-                    fieldtype = fixup_ctype(fieldtype, fieldname, (size, sign))
-                layout_addfield(layout, offset, fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(fieldtype))
-
-        # try to enforce the same alignment as the one of the original
-        # structure
-        if alignment < info['align']:
-            choices = [ctype for ctype in alignment_types
-                             if ctype_alignment(ctype) == info['align']]
-            assert choices, "unsupported alignment %d" % (info['align'],)
-            choices = [(ctypes.sizeof(ctype), i, ctype)
-                       for i, ctype in enumerate(choices)]
-            csize, _, ctype = min(choices)
-            for i in range(0, info['size'] - csize + 1, info['align']):
-                if layout[i:i+csize] == [None] * csize:
-                    layout_addfield(layout, i, ctype, '_alignment')
-                    break
-            else:
-                raise AssertionError("unenforceable alignment %d" % (
-                    info['align'],))
-
-        n = 0
-        for i, cell in enumerate(layout):
-            if cell is not None:
-                continue
-            layout_addfield(layout, i, ctypes.c_char, '_pad%d' % (n,))
-            n += 1
-
-        # build the ctypes Structure
-        seen = {}
-        fields = []
-        for cell in layout:
-            if cell in seen:
-                continue
-            fields.append((cell.name, cell.ctype))
-            seen[cell] = True
-
-        class S(ctypes.Structure):
-            _fields_ = fields
-        name = self.name
-        if name.startswith('struct '):
-            name = name[7:]
-        S.__name__ = name
-        return S
-
-class SimpleType(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined simple numeric type.
-    """
-    def __init__(self, name, ctype_hint=ctypes.c_int, ifdef=None):
-        self.name = name
-        self.ctype_hint = ctype_hint
-        self.ifdef = ifdef
-        
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield ''
-        yield 'ctypesplatcheck_t x;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        if self.ctype_hint in integer_class:
-            yield 'x = 0; x = ~x;'
-            yield 'dump("unsigned", x > 0);'
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None and not info['defined']:
-            return None
-        size = info['size']
-        sign = info.get('unsigned', False)
-        ctype = self.ctype_hint
-        if (size, sign) != size_and_sign(ctype):
-            ctype = fixup_ctype(ctype, self.name, (size, sign))
-        return ctype
-
-class ConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant.
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
-        yield '    long long x = (long long)(%s);' % (self.name,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.name,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-
-    def build_result(self, info, config_result):
-        return info['value']
-
-class DefinedConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant. If not #defined the value will be None.
-    """
-    def __init__(self, macro):
-        self.name = self.macro = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
-        yield '    long long x = (long long)(%s);' % (self.macro,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.macro,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            return info['value']
-        return None
-
-
-class DefinedConstantString(CConfigEntry):
-    """
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'int i;'
-        yield 'char *p = %s;' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'for (i = 0; p[i] != 0; i++ ) {'
-        yield '  printf("value_%d: %d\\n", i, (int)(unsigned char)p[i]);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            string = ''
-            d = 0
-            while info.has_key('value_%d' % d):
-                string += chr(info['value_%d' % d])
-                d += 1
-            return string
-        return None
-
-
-class Defined(CConfigEntry):
-    """A boolean, corresponding to an #ifdef.
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % (self.macro,)
-        yield 'dump("defined", 1);'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        return bool(info['defined'])
-
-class CConfigSingleEntry(object):
-    """ An abstract class of type which requires
-    gcc succeeding/failing instead of only asking
-    """
-    pass
-
-class Has(CConfigSingleEntry):
-    def __init__(self, name):
-        self.name = name
-    
-    def question(self, ask_gcc):
-        return ask_gcc(self.name + ';')
-
-class Works(CConfigSingleEntry):
-    def question(self, ask_gcc):
-        return ask_gcc("")
-
-class SizeOf(CConfigEntry):
-    """An entry in a CConfig class that stands for
-    some external opaque type
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'dump("size",  sizeof(%s));' % self.name
-
-    def build_result(self, info, config_result):
-        return info['size']
-
-# ____________________________________________________________
-#
-# internal helpers
-
-def ctype_alignment(c_type):
-    if issubclass(c_type, ctypes.Structure):
-        return max([ctype_alignment(fld_type)
-                     for fld_name, fld_type in c_type._fields_])
-    
-    return ctypes.alignment(c_type)
-
-def uniquefilepath(LAST=[0]):
-    i = LAST[0]
-    LAST[0] += 1
-    return configdir.join('ctypesplatcheck_%d.c' % i)
-
-alignment_types = [
-    ctypes.c_short,
-    ctypes.c_int,
-    ctypes.c_long,
-    ctypes.c_float,
-    ctypes.c_double,
-    ctypes.c_char_p,
-    ctypes.c_void_p,
-    ctypes.c_longlong,
-    ctypes.c_wchar,
-    ctypes.c_wchar_p,
-    ]
-
-integer_class = [ctypes.c_byte,     ctypes.c_ubyte,
-                 ctypes.c_short,    ctypes.c_ushort,
-                 ctypes.c_int,      ctypes.c_uint,
-                 ctypes.c_long,     ctypes.c_ulong,
-                 ctypes.c_longlong, ctypes.c_ulonglong,
-                 ]
-float_class = [ctypes.c_float, ctypes.c_double]
-
-class Field(object):
-    def __init__(self, name, ctype):
-        self.name = name
-        self.ctype = ctype
-    def __repr__(self):
-        return '<field %s: %s>' % (self.name, self.ctype)
-
-def layout_addfield(layout, offset, ctype, prefix):
-    size = ctypes.sizeof(ctype)
-    name = prefix
-    i = 0
-    while name in layout:
-        i += 1
-        name = '%s_%d' % (prefix, i)
-    field = Field(name, ctype)
-    for i in range(offset, offset+size):
-        assert layout[i] is None, "%s overlaps %r" % (fieldname, layout[i])
-        layout[i] = field
-    return field
-
-def size_and_sign(ctype):
-    return (ctypes.sizeof(ctype),
-            ctype in integer_class and ctype(-1).value > 0)
-
-def fixup_ctype(fieldtype, fieldname, expected_size_and_sign):
-    for typeclass in [integer_class, float_class]:
-        if fieldtype in typeclass:
-            for ctype in typeclass:
-                if size_and_sign(ctype) == expected_size_and_sign:
-                    return ctype
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_char):
-        # for now, assume it is an array of chars; otherwise we'd also
-        # have to check the exact integer type of the elements of the array
-        size, sign = expected_size_and_sign
-        return ctypes.c_char * size
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_ubyte):
-        # grumble, fields of type 'c_char array' have automatic cast-to-
-        # Python-string behavior in ctypes, which may not be what you
-        # want, so here is the same with c_ubytes instead...
-        size, sign = expected_size_and_sign
-        return ctypes.c_ubyte * size
-    raise TypeError("conflicting field type %r for %r" % (fieldtype,
-                                                          fieldname))
-
-
-C_HEADER = """
-#include <stdio.h>
-#include <stddef.h>   /* for offsetof() */
-#ifndef _WIN32
-#  include <stdint.h>   /* FreeBSD: for uint64_t */
-#endif
-
-void dump(char* key, int value) {
-    printf("%s: %d\\n", key, value);
-}
-"""
-
-def run_example_code(filepath, eci, noerr=False):
-    executable = build_executable([filepath], eci, noerr=noerr)
-    output = py.process.cmdexec(executable)
-    section = None
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith('-+- '):      # start of a new section
-            section = {}
-        elif line == '---':              # section end
-            assert section is not None
-            yield section
-            section = None
-        elif line:
-            assert section is not None
-            key, value = line.split(': ')
-            section[key] = int(value)
-
-# ____________________________________________________________
-
-def get_python_include_dir():
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    return gcv['INCLUDEPY']
-
-if __name__ == '__main__':
-    doc = """Example:
-    
-       ctypes_platform.py  -h sys/types.h  -h netinet/in.h
-                           'struct sockaddr_in'
-                           sin_port  c_int
-    """
-    import sys, getopt
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'h:')
-    if not args:
-        print >> sys.stderr, doc
-    else:
-        assert len(args) % 2 == 1
-        headers = []
-        for opt, value in opts:
-            if opt == '-h':
-                headers.append('#include <%s>' % (value,))
-        name = args[0]
-        fields = []
-        for i in range(1, len(args), 2):
-            ctype = getattr(ctypes, args[i+1])
-            fields.append((args[i], ctype))
-
-        S = getstruct(name, '\n'.join(headers), fields)
-
-        for key, value in S._fields_:
-            print key, value
diff --git a/ctypes_configure/doc/configure.html b/ctypes_configure/doc/configure.html
deleted file mode 100644
--- a/ctypes_configure/doc/configure.html
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="latin1" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=latin1" />
-<meta name="generator" content="Docutils 0.4.1: http://docutils.sourceforge.net/" />
-<title>ctypes configure</title>
-</head>
-<body>
-<div class="document" id="ctypes-configure">
-<h1 class="title">ctypes configure</h1>
-<div class="section">
-<h1><a id="idea" name="idea">idea</a></h1>
-<p>One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).</p>
-</div>
-<div class="section">
-<h1><a id="installation" name="installation">installation</a></h1>
-<p><tt class="docutils literal"><span class="pre">easy_install</span> <span class="pre">ctypes_configure</span></tt></p>
-</div>
-<div class="section">
-<h1><a id="usage" name="usage">usage</a></h1>
-<p><a class="reference" href="http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py">sample.py</a> explains in details how to use it.</p>
-</div>
-</div>
-</body>
-</html>
diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt
deleted file mode 100644
--- a/ctypes_configure/doc/configure.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-=================
-ctypes configure
-=================
-
-idea
-====
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc
-for various platform-dependent details like
-exact sizes of types (for example size\_t), #defines, exact outline
-of structures etc. It replaces in this regard code generator (h2py).
-
-installation
-============
-
-``easy_install ctypes_configure``
-
-usage
-=====
-
-:source:`sample.py <ctypes_configure/doc/sample.py>` explains in details how to use it.
diff --git a/ctypes_configure/doc/sample.py b/ctypes_configure/doc/sample.py
deleted file mode 100644
--- a/ctypes_configure/doc/sample.py
+++ /dev/null
@@ -1,72 +0,0 @@
-
-from ctypes_configure import configure
-import ctypes
-
-class CConfigure:
-    _compilation_info_ = configure.ExternalCompilationInfo(
-        
-        # all lines landing in C header before includes
-        pre_include_lines = [],
-
-        # list of .h files to include
-        includes = ['time.h', 'sys/time.h', 'unistd.h'],
-
-        # list of directories to search for include files
-        include_dirs = [],
-
-        # all lines landing in C header after includes
-        post_include_lines = [],
-
-        # libraries to link with
-        libraries = [],
-
-        # library directories
-        library_dirs = [],
-
-        # additional C sources to compile with (that go to
-        # created .c files)
-        separate_module_sources = [],
-
-        # additional existing C source file names
-        separate_module_files = [],
-        )
-
-    # get real int type out of hint and name
-    size_t = configure.SimpleType('size_t', ctypes.c_int)
-
-    # grab value of numerical #define
-    NULL = configure.ConstantInteger('NULL')
-
-    # grab #define, whether it's defined or not
-    EXISTANT = configure.Defined('NULL')
-    NOT_EXISTANT = configure.Defined('XXXNOTNULL')
-
-    # check for existance of C functions
-    has_write = configure.Has('write')
-    no_xxxwrite = configure.Has('xxxwrite')
-
-    # check for size of type
-    sizeof_size_t = configure.SizeOf('size_t')
-
-    # structure, with given hints for interesting fields,
-    # types does not need to be too specific.
-    # all interesting fields would end up with right offset
-    # size and order
-    struct_timeval = configure.Struct('struct timeval',[
-        ('tv_sec', ctypes.c_int),
-        ('tv_usec', ctypes.c_int)])
-
-info = configure.configure(CConfigure)
-
-assert info['has_write']
-assert not info['no_xxxwrite']
-assert info['NULL'] == 0
-size_t = info['size_t']
-print "size_t in ctypes is ", size_t
-assert ctypes.sizeof(size_t) == info['sizeof_size_t']
-assert info['EXISTANT']
-assert not info['NOT_EXISTANT']
-print
-print "fields of struct timeval are "
-for name, value in info['struct_timeval']._fields_:
-    print "  ", name, " ", value
diff --git a/ctypes_configure/dumpcache.py b/ctypes_configure/dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/dumpcache.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os, sys
-import ctypes
-
-
-def dumpcache(referencefilename, filename, config):
-    dirname = os.path.dirname(referencefilename)
-    filename = os.path.join(dirname, filename)
-    f = open(filename, 'w')
-    print >> f, 'import ctypes'
-    print >> f
-    names = config.keys()
-    names.sort()
-    print >> f, '__all__ = %r' % (tuple(names),)
-    print >> f
-    for key in names:
-        val = config[key]
-        if isinstance(val, (int, long)):
-            f.write("%s = %d\n" % (key, val))
-        elif val is None:
-            f.write("%s = None\n" % key)
-        elif isinstance(val, ctypes.Structure.__class__):
-            f.write("class %s(ctypes.Structure):\n" % key)
-            f.write("    _fields_ = [\n")
-            for k, v in val._fields_:
-                f.write("        ('%s', %s),\n" % (k, ctypes_repr(v)))
-            f.write("    ]\n")
-        elif isinstance(val, (tuple, list)):
-            for x in val:
-                assert isinstance(x, (int, long, str)), \
-                       "lists of integers or strings only"
-            f.write("%s = %r\n" % (key, val))
-        else:
-            # a simple type, hopefully
-            f.write("%s = %s\n" % (key, ctypes_repr(val)))
-    f.close()
-    print 'Wrote %s.' % (filename,)
-    sys.stdout.flush()
-
-def ctypes_repr(cls):
-    # ctypes_configure does not support nested structs so far
-    # so let's ignore it
-    if isinstance(cls, ctypes._SimpleCData.__class__):
-        return "ctypes." + cls.__name__
-    if hasattr(cls, '_length_') and hasattr(cls, '_type_'):  # assume an array
-        return '%s*%d' % (ctypes_repr(cls._type_), cls._length_)
-    raise NotImplementedError("saving of object with type %r" % type(cls))
diff --git a/ctypes_configure/stdoutcapture.py b/ctypes_configure/stdoutcapture.py
deleted file mode 100644
--- a/ctypes_configure/stdoutcapture.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-A quick hack to capture stdout/stderr.
-"""
-
-import os, sys
-
-
-class Capture:
-    
-    def __init__(self, mixed_out_err = False):
-        "Start capture of the Unix-level stdout and stderr."
-        if (not hasattr(os, 'tmpfile') or
-            not hasattr(os, 'dup') or
-            not hasattr(os, 'dup2') or
-            not hasattr(os, 'fdopen')):
-            self.dummy = 1
-        else:
-            try:
-                self.tmpout = os.tmpfile()
-                if mixed_out_err:
-                    self.tmperr = self.tmpout
-                else:
-                    self.tmperr = os.tmpfile()
-            except OSError:     # bah?  on at least one Windows box
-                self.dummy = 1
-                return
-            self.dummy = 0
-            # make new stdout/stderr files if needed
-            self.localoutfd = os.dup(1)
-            self.localerrfd = os.dup(2)
-            if hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == 1:
-                self.saved_stdout = sys.stdout
-                sys.stdout = os.fdopen(self.localoutfd, 'w', 1)
-            else:
-                self.saved_stdout = None
-            if hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == 2:
-                self.saved_stderr = sys.stderr
-                sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
-            else:
-                self.saved_stderr = None
-            os.dup2(self.tmpout.fileno(), 1)
-            os.dup2(self.tmperr.fileno(), 2)
-
-    def done(self):
-        "End capture and return the captured text (stdoutfile, stderrfile)."
-        if self.dummy:
-            import cStringIO
-            return cStringIO.StringIO(), cStringIO.StringIO()
-        else:
-            os.dup2(self.localoutfd, 1)
-            os.dup2(self.localerrfd, 2)
-            if self.saved_stdout is not None:
-                f = sys.stdout
-                sys.stdout = self.saved_stdout
-                f.close()
-            else:
-                os.close(self.localoutfd)
-            if self.saved_stderr is not None:
-                f = sys.stderr
-                sys.stderr = self.saved_stderr
-                f.close()
-            else:
-                os.close(self.localerrfd)
-            self.tmpout.seek(0)
-            self.tmperr.seek(0)
-            return self.tmpout, self.tmperr
-
-
-if __name__ == '__main__':
-    # test
-    c = Capture()
-    try:
-        os.system('echo hello')
-    finally:
-        fout, ferr = c.done()
-    print 'Output:', `fout.read()`
-    print 'Error:', `ferr.read()`
diff --git a/ctypes_configure/test/__init__.py b/ctypes_configure/test/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/test/test_configure.py b/ctypes_configure/test/test_configure.py
deleted file mode 100644
--- a/ctypes_configure/test/test_configure.py
+++ /dev/null
@@ -1,212 +0,0 @@
-import py, sys, struct
-from ctypes_configure import configure
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import ctypes
-
-def test_dirent():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_reclen", ctypes.c_ushort)])
-    assert issubclass(dirent, ctypes.Structure)
-    ssize = (ctypes.sizeof(ctypes.c_long) +
-             ctypes.sizeof(ctypes.c_int) +
-             ctypes.sizeof(ctypes.c_ushort) +
-             32)
-    extra_padding = (-ssize) % ctypes.alignment(ctypes.c_long)
-
-    assert dirent._fields_ == [('_alignment', ctypes.c_long),
-                               ('_pad0', ctypes.c_char),
-                               ('_pad1', ctypes.c_char),
-                               ('_pad2', ctypes.c_char),
-                               ('_pad3', ctypes.c_char),
-                               ('d_reclen', ctypes.c_ushort),
-                               ] + [
-                               ('_pad%d' % n, ctypes.c_char)
-                                    for n in range(4, 4+32+extra_padding)]
-    assert ctypes.sizeof(dirent) == ssize + extra_padding
-    assert ctypes.alignment(dirent) == ctypes.alignment(ctypes.c_long)
-
-def test_fit_type():
-    S = configure.getstruct("struct S",
-                                  """
-           struct S {
-               signed char c;
-               unsigned char uc;
-               short s;
-               unsigned short us;
-               int i;
-               unsigned int ui;
-               long l;
-               unsigned long ul;
-               long long ll;
-               unsigned long long ull;
-               float f;
-               double d;
-           };
-                                  """,
-                                  [("c",   ctypes.c_int),
-                                   ("uc",  ctypes.c_int),
-                                   ("s",   ctypes.c_uint),
-                                   ("us",  ctypes.c_int),
-                                   ("i",   ctypes.c_int),
-                                   ("ui",  ctypes.c_int),
-                                   ("l",   ctypes.c_int),
-                                   ("ul",  ctypes.c_int),
-                                   ("ll",  ctypes.c_int),
-                                   ("ull", ctypes.c_int),
-                                   ("f",   ctypes.c_double),
-                                   ("d",   ctypes.c_float)])
-    assert issubclass(S, ctypes.Structure)
-    fields = dict(S._fields_)
-    assert fields["c"] == ctypes.c_byte
-    assert fields["uc"] == ctypes.c_ubyte
-    assert fields["s"] == ctypes.c_short
-    assert fields["us"] == ctypes.c_ushort
-    assert fields["i"] == ctypes.c_int
-    assert fields["ui"] == ctypes.c_uint
-    assert fields["l"] == ctypes.c_long
-    assert fields["ul"] == ctypes.c_ulong
-    assert fields["ll"] == ctypes.c_longlong
-    assert fields["ull"] == ctypes.c_ulonglong
-    assert fields["f"] == ctypes.c_float
-    assert fields["d"] == ctypes.c_double
-
-def test_simple_type():
-    ctype = configure.getsimpletype('test_t',
-                                          'typedef unsigned short test_t;',
-                                          ctypes.c_int)
-    assert ctype == ctypes.c_ushort
-
-def test_constant_integer():
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (6*7)')
-    assert value == 42
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (-2147483648LL)')
-    assert value == -2147483648
-    value = configure.getconstantinteger('BLAH',
-                                               '#define BLAH (3333333333ULL)')
-    assert value == 3333333333
-
-def test_defined():
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE', '')
-    assert not res
-    res = configure.getdefined('ALFKJLKJFLKJFKLEJDLKEWMECEE',
-                                     '#define ALFKJLKJFLKJFKLEJDLKEWMECEE')
-    assert res
-
-def test_configure():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform.h')
-    test_h.write('#define XYZZY 42\n')
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-
-    res = configure.configure(CConfig)
-    assert issubclass(res['FILE'], ctypes.Structure)
-    assert res == {'FILE': res['FILE'],
-                   'ushort': ctypes.c_ushort,
-                   'XYZZY': 42}
-
-def test_ifdef():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines = ['/* a C comment */',
-                                  '#define XYZZY 42',
-                                  'typedef int foo;',
-                                  'struct s {',
-                                  'int i;',
-                                  'double f;'
-                                  '};'])
-
-
-        s = configure.Struct('struct s', [('i', ctypes.c_int)],
-                                   ifdef='XYZZY')
-        z = configure.Struct('struct z', [('i', ctypes.c_int)],
-                                   ifdef='FOOBAR')
-
-        foo = configure.SimpleType('foo', ifdef='XYZZY')
-        bar = configure.SimpleType('bar', ifdef='FOOBAR')
-
-    res = configure.configure(CConfig)
-    assert res['s'] is not None
-    assert res['z'] is None
-    assert res['foo'] is not None
-    assert res['bar'] is None
-
-def test_nested_structs():
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            post_include_lines="""
-            struct x {
-            int foo;
-            unsigned long bar;
-            };
-            struct y {
-            char c;
-            struct x x;
-            };
-            """.split("\n"))
-
-        x = configure.Struct("struct x", [("bar", ctypes.c_short)])
-        y = configure.Struct("struct y", [("x", x)])
-
-    res = configure.configure(CConfig)
-    c_x = res["x"]
-    c_y = res["y"]
-    c_y_fields = dict(c_y._fields_)
-    assert issubclass(c_x , ctypes.Structure)
-    assert issubclass(c_y, ctypes.Structure)
-    assert c_y_fields["x"] is c_x
-
-def test_array():
-    dirent = configure.getstruct("struct dirent",
-                                       """
-           struct dirent  /* for this example only, not the exact dirent */
-           {
-               long d_ino;
-               int d_off;
-               unsigned short d_reclen;
-               char d_name[32];
-           };
-                                       """,
-                                       [("d_name", ctypes.c_char * 0)])
-    assert dirent.d_name.size == 32
-
-def test_has():
-    assert configure.has("x", "int x = 3;")
-    assert not configure.has("x", "")
-    # has() should also not crash if it is given an invalid #include
-    assert not configure.has("x", "#include <some/path/which/cannot/exist>")
-
-def test_check_eci():
-    eci = ExternalCompilationInfo()
-    assert configure.check_eci(eci)
-    eci = ExternalCompilationInfo(libraries=['some_name_that_doesnt_exist_'])
-    assert not configure.check_eci(eci)
-
-def test_sizeof():
-    assert configure.sizeof("char", ExternalCompilationInfo()) == 1
-
-def test_memory_alignment():
-    a = configure.memory_alignment()
-    print a
-    assert a % struct.calcsize("P") == 0
diff --git a/ctypes_configure/test/test_dumpcache.py b/ctypes_configure/test/test_dumpcache.py
deleted file mode 100644
--- a/ctypes_configure/test/test_dumpcache.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import ctypes
-from ctypes_configure import configure, dumpcache
-from ctypes_configure.cbuild import ExternalCompilationInfo
-
-
-def test_cache():
-    configdir = configure.configdir
-    test_h = configdir.join('test_ctypes_platform2.h')
-    test_h.write('#define XYZZY 42\n'
-                 "#define large 2147483648L\n")
-
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            pre_include_lines = ["/* a C comment */",
-                                 "#include <stdio.h>",
-                                 "#include <test_ctypes_platform2.h>"],
-            include_dirs = [str(configdir)]
-        )
-
-        FILE = configure.Struct('FILE', [])
-        ushort = configure.SimpleType('unsigned short')
-        XYZZY = configure.ConstantInteger('XYZZY')
-        XUZ = configure.Has('XUZ')
-        large = configure.DefinedConstantInteger('large')
-        undef = configure.Defined('really_undefined')
-
-    res = configure.configure(CConfig)
-
-    cachefile = configdir.join('cache')
-    dumpcache.dumpcache('', str(cachefile), res)
-
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['XYZZY'] == res['XYZZY']
-    assert d['ushort'] == res['ushort']
-    assert d['FILE']._fields_ == res['FILE']._fields_
-    assert d['FILE'].__mro__[1:] == res['FILE'].__mro__[1:]
-    assert d['undef'] == res['undef']
-    assert d['large'] == res['large']
-    assert d['XUZ'] == res['XUZ']
-
-
-def test_cache_array():
-    configdir = configure.configdir
-    res = {'foo': ctypes.c_short * 27}
-    cachefile = configdir.join('cache_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
-
-def test_cache_array_array():
-    configdir = configure.configdir
-    res = {'foo': (ctypes.c_int * 2) * 3}
-    cachefile = configdir.join('cache_array_array')
-    dumpcache.dumpcache('', str(cachefile), res)
-    #
-    d = {}
-    execfile(str(cachefile), d)
-    assert d['foo'] == res['foo']
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -33,6 +33,10 @@
     from __pypy__ import reversed_dict as _reversed_dict
 except ImportError:
     _reversed_dict = None     # don't have ordered dicts
+try:
+    from __pypy__ import dict_popitem_first as _dict_popitem_first
+except ImportError:
+    _dict_popitem_first = None
 
 try:
     from thread import get_ident as _get_ident
@@ -44,6 +48,17 @@
 ### OrderedDict
 ################################################################################
 
+if _dict_popitem_first is None:
+    def _dict_popitem_first(self):
+        it = dict.iteritems(self)
+        try:
+            k, v = it.next()
+        except StopIteration:
+            raise KeyError('dictionary is empty')
+        dict.__delitem__(self, k)
+        return (k, v)
+
+
 class OrderedDict(dict):
     '''Dictionary that remembers insertion order.
 
@@ -68,12 +83,7 @@
         if last:
             return dict.popitem(self)
         else:
-            it = dict.__iter__(self)
-            try:
-                k = it.next()
-            except StopIteration:
-                raise KeyError('dictionary is empty')
-            return (k, self.pop(k))
+            return _dict_popitem_first(self)
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises((TypeError, ValueError),
+        self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
     def test_fom_buffer_with_offset(self):
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -310,6 +310,10 @@
                 static = os.path.join(sysroot, dir[1:], static_f)
                 xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f)
 
+            # PyPy extension here: 'shared' usually ends in something
+            # like '.pypy-41.so'.  Try without the '.pypy-41' part too.
+            shared_no_pypy = re.sub(r'[.]pypy[^.]+([.][^.]+)$', r'\1', shared)
+
             # We're second-guessing the linker here, with not much hard
             # data to go on: GCC seems to prefer the shared library, so I'm
             # assuming that *all* Unix C compilers do.  And of course I'm
@@ -320,6 +324,8 @@
                 return xcode_stub
             elif os.path.exists(shared):
                 return shared
+            elif os.path.exists(shared_no_pypy):
+                return shared_no_pypy
             elif os.path.exists(static):
                 return static
 
diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py
--- a/lib-python/2.7/test/test_capi.py
+++ b/lib-python/2.7/test/test_capi.py
@@ -26,7 +26,6 @@
 skips = []
 if support.check_impl_detail(pypy=True):
     skips += [
-            'test_broken_memoryview',
             'test_buildvalue_N',
             'test_capsule',
             'test_lazy_hash_inheritance',
diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1969,9 +1969,10 @@
         if not gc.isenabled():
             gc.enable()
             self.addCleanup(gc.disable)
-        #thresholds = gc.get_threshold()
-        #self.addCleanup(gc.set_threshold, *thresholds)
-        #gc.set_threshold(10)
+        if test_support.check_impl_detail(cpython=True):
+            thresholds = gc.get_threshold()
+            self.addCleanup(gc.set_threshold, *thresholds)
+            gc.set_threshold(10)
 
         # perform numerous block allocations, with cyclic references to make
         # sure objects are collected asynchronously by the gc
diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -31,6 +31,26 @@
            "WeakKeyDictionary", "ReferenceError", "ReferenceType", "ProxyType",
            "CallableProxyType", "ProxyTypes", "WeakValueDictionary", 'WeakSet']
 
+try:
+    from __pypy__ import delitem_if_value_is as _delitem_if_value_is
+except ImportError:
+    def _delitem_if_value_is(d, key, value):
+        try:
+            if self.data[key] is value:  # fall-back: there is a potential
+                #             race condition in multithreaded programs HERE
+                del self.data[key]
+        except KeyError:
+            pass
+
+def _remove_dead_weakref(d, key):
+    try:
+        wr = d[key]
+    except KeyError:
+        pass
+    else:
+        if wr() is None:
+            _delitem_if_value_is(d, key, wr)
+
 
 class WeakValueDictionary(UserDict.UserDict):
     """Mapping class that references values weakly.
@@ -58,14 +78,9 @@
                 if self._iterating:
                     self._pending_removals.append(wr.key)
                 else:
-                    # Changed this for PyPy: made more resistent.  The
-                    # issue is that in some corner cases, self.data
-                    # might already be changed or removed by the time
-                    # this weakref's callback is called.  If that is
-                    # the case, we don't want to randomly kill an
-                    # unrelated entry.
-                    if self.data.get(wr.key) is wr:
-                        del self.data[wr.key]
+                    # Atomic removal is necessary since this function
+                    # can be called asynchronously by the GC
+                    _delitem_if_value_is(self.data, wr.key, wr)
         self._remove = remove
         # A list of keys to be removed
         self._pending_removals = []
@@ -78,7 +93,8 @@
         # We shouldn't encounter any KeyError, because this method should
         # always be called *before* mutating the dict.
         while l:
-            del d[l.pop()]
+            key = l.pop()
+            _remove_dead_weakref(d, key)
 
     def __getitem__(self, key):
         o = self.data[key]()
diff --git a/lib-python/3/_collections_abc.py b/lib-python/3/_collections_abc.py
--- a/lib-python/3/_collections_abc.py
+++ b/lib-python/3/_collections_abc.py
@@ -29,8 +29,8 @@
 # so that they will pass tests like:
 #       it = iter(somebytearray)
 #       assert isinstance(it, Iterable)
-# Note:  in other implementations, these types many not be distinct
-# and they make have their own implementation specific types that
+# Note:  in other implementations, these types might not be distinct
+# and they may have their own implementation specific types that
 # are not included on this list.
 bytes_iterator = type(iter(b''))
 bytearray_iterator = type(iter(bytearray()))
@@ -41,6 +41,7 @@
 list_iterator = type(iter([]))
 list_reverseiterator = type(iter(reversed([])))
 range_iterator = type(iter(range(0)))
+longrange_iterator = type(iter(range(1 << 1000)))
 set_iterator = type(iter(set()))
 str_iterator = type(iter(""))
 tuple_iterator = type(iter(()))
@@ -234,6 +235,7 @@
 Iterator.register(list_iterator)
 Iterator.register(list_reverseiterator)
 Iterator.register(range_iterator)
+Iterator.register(longrange_iterator)
 Iterator.register(set_iterator)
 Iterator.register(str_iterator)
 Iterator.register(tuple_iterator)
diff --git a/lib-python/3/_pydecimal.py b/lib-python/3/_pydecimal.py
--- a/lib-python/3/_pydecimal.py
+++ b/lib-python/3/_pydecimal.py
@@ -1068,12 +1068,11 @@
         return sign + intpart + fracpart + exp
 
     def to_eng_string(self, context=None):
-        """Convert to engineering-type string.
-
-        Engineering notation has an exponent which is a multiple of 3, so there
-        are up to 3 digits left of the decimal place.
-
-        Same rules for when in exponential and when as a value as in __str__.
+        """Convert to a string, using engineering notation if an exponent is needed.
+
+        Engineering notation has an exponent which is a multiple of 3.  This
+        can leave up to 3 digits to the left of the decimal place and may
+        require the addition of either one or two trailing zeros.
         """
         return self.__str__(eng=True, context=context)
 
@@ -4107,7 +4106,7 @@
         >>> context.create_decimal_from_float(3.1415926535897932)
         Traceback (most recent call last):
             ...
-        decimal.Inexact
+        decimal.Inexact: None
 
         """
         d = Decimal.from_float(f)       # An exact conversion
@@ -5502,9 +5501,29 @@
             return r
 
     def to_eng_string(self, a):
-        """Converts a number to a string, using scientific notation.
+        """Convert to a string, using engineering notation if an exponent is needed.
+
+        Engineering notation has an exponent which is a multiple of 3.  This
+        can leave up to 3 digits to the left of the decimal place and may
+        require the addition of either one or two trailing zeros.
 
         The operation is not affected by the context.
+
+        >>> ExtendedContext.to_eng_string(Decimal('123E+1'))
+        '1.23E+3'
+        >>> ExtendedContext.to_eng_string(Decimal('123E+3'))
+        '123E+3'
+        >>> ExtendedContext.to_eng_string(Decimal('123E-10'))
+        '12.3E-9'
+        >>> ExtendedContext.to_eng_string(Decimal('-123E-12'))
+        '-123E-12'
+        >>> ExtendedContext.to_eng_string(Decimal('7E-7'))
+        '700E-9'
+        >>> ExtendedContext.to_eng_string(Decimal('7E+1'))
+        '70'
+        >>> ExtendedContext.to_eng_string(Decimal('0E+1'))
+        '0.00E+3'
+
         """
         a = _convert_other(a, raiseit=True)
         return a.to_eng_string(context=self)
diff --git a/lib-python/3/_pyio.py b/lib-python/3/_pyio.py
--- a/lib-python/3/_pyio.py
+++ b/lib-python/3/_pyio.py
@@ -276,7 +276,7 @@
 try:
     UnsupportedOperation = io.UnsupportedOperation
 except AttributeError:
-    class UnsupportedOperation(ValueError, OSError):
+    class UnsupportedOperation(OSError, ValueError):
         pass
 
 
diff --git a/lib-python/3/antigravity.py b/lib-python/3/antigravity.py
--- a/lib-python/3/antigravity.py
+++ b/lib-python/3/antigravity.py
@@ -2,7 +2,7 @@
 import webbrowser
 import hashlib
 
-webbrowser.open("http://xkcd.com/353/")
+webbrowser.open("https://xkcd.com/353/")
 
 def geohash(latitude, longitude, datedow):
     '''Compute geohash() using the Munroe algorithm.
diff --git a/lib-python/3/asyncio/base_events.py b/lib-python/3/asyncio/base_events.py
--- a/lib-python/3/asyncio/base_events.py
+++ b/lib-python/3/asyncio/base_events.py
@@ -13,7 +13,6 @@
 to modify the meaning of the API call itself.
 """
 
-
 import collections
 import concurrent.futures
 import heapq
@@ -28,6 +27,7 @@
 import traceback
 import sys
 import warnings
+import weakref
 
 from . import compat
 from . import coroutines
@@ -41,9 +41,6 @@
 __all__ = ['BaseEventLoop']
 
 
-# Argument for default thread pool executor creation.
-_MAX_WORKERS = 5
-
 # Minimum number of _scheduled timer handles before cleanup of
 # cancelled handles is performed.
 _MIN_SCHEDULED_TIMER_HANDLES = 100
@@ -76,12 +73,29 @@
         return repr(fd)
 
 
-# Linux's sock.type is a bitmask that can include extra info about socket.
-_SOCKET_TYPE_MASK = 0
-if hasattr(socket, 'SOCK_NONBLOCK'):
-    _SOCKET_TYPE_MASK |= socket.SOCK_NONBLOCK
-if hasattr(socket, 'SOCK_CLOEXEC'):
-    _SOCKET_TYPE_MASK |= socket.SOCK_CLOEXEC
+def _set_reuseport(sock):
+    if not hasattr(socket, 'SO_REUSEPORT'):
+        raise ValueError('reuse_port not supported by socket module')
+    else:
+        try:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        except OSError:
+            raise ValueError('reuse_port not supported by socket module, '
+                             'SO_REUSEPORT defined but not implemented.')
+
+
+def _is_stream_socket(sock):
+    # Linux's socket.type is a bitmask that can include extra info
+    # about socket, therefore we can't do simple
+    # `sock_type == socket.SOCK_STREAM`.
+    return (sock.type & socket.SOCK_STREAM) == socket.SOCK_STREAM
+
+
+def _is_dgram_socket(sock):
+    # Linux's socket.type is a bitmask that can include extra info
+    # about socket, therefore we can't do simple

From pypy.commits at gmail.com  Mon Feb 13 10:12:11 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 13 Feb 2017 07:12:11 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: set flag to identify a jit frame
 by libunwind's proc info
Message-ID: <58a1cccb.84e11c0a.50880.c84c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90087:40045929f6e3
Date: 2017-02-13 15:12 +0100
http://bitbucket.org/pypy/pypy/changeset/40045929f6e3/

Log:	set flag to identify a jit frame by libunwind's proc info

diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c
@@ -100,7 +100,9 @@
     unw_dyn_proc_info_t * ip = (unw_dyn_proc_info_t*)&(u->u);
     ip->name_ptr = (unw_word_t)name_cpy;
     ip->handler = 0;
-    ip->flags = 0;
+    // the docs say, we cannot use this field. but looking at libunwind, it just copies
+    // the value over when unw_get_proc_info is called. This should be fine to identify
+    ip->flags = DYN_JIT_FLAG;
     ip->regions = NULL;
 
     _U_dyn_register(u);
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
--- a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
+++ b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
@@ -5,6 +5,8 @@
 
 #include "rvmprof.h"
 
+#define DYN_JIT_FLAG 0xbeefbeef
+
 RPY_EXTERN int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr,
                               const char * name);
 RPY_EXTERN int vmp_dyn_cancel(int ref);
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -253,7 +253,8 @@
             // this is possible because compiler align to 8 bytes.
             //
 #ifdef PYPY_JIT_CODEMAP
-            if (pip.format == UNW_INFO_FORMAT_DYNAMIC) {
+            // see vmp_dynamic.c on line ip->flags = DYN_JIT_FLAG
+            if (pip.flags == DYN_JIT_FLAG) {
                 if (top_most_frame->kind != VMPROF_JITTED_TAG) {
                     // if this is encountered frequently something is wrong with
                     // the stack building

From pypy.commits at gmail.com  Mon Feb 13 10:12:13 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 13 Feb 2017 07:12:13 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: remove static declaration
Message-ID: <58a1cccd.4a86df0a.82409.bf68@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90088:254b4656b2e4
Date: 2017-02-13 15:32 +0100
http://bitbucket.org/pypy/pypy/changeset/254b4656b2e4/

Log:	remove static declaration

diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -95,13 +95,21 @@
         char interp_name[259];
     } header;
 
+    const char * machine;
+
+    machine = vmp_machine_os_name();
+
     size_t namelen = strnlen(interp_name, 255);
 
     header.hdr[0] = 0;
     header.hdr[1] = 3;
     header.hdr[2] = 0;
     header.hdr[3] = prepare_interval_usec;
-    header.hdr[4] = 0;
+    if (strstr(machine, "win") != 0) {
+        header.hdr[4] = 1;
+    } else {
+        header.hdr[4] = 0;
+    }
     header.interp_name[0] = MARKER_HEADER;
     header.interp_name[1] = '\x00';
     header.interp_name[2] = VERSION_TIMESTAMP;
@@ -122,7 +130,7 @@
     (void)vmp_write_time_now(MARKER_TIME_N_ZONE);
 
     /* write some more meta information */
-    vmp_write_meta("os", vmp_machine_os_name());
+    vmp_write_meta("os", machine);
     bits = vmp_machine_bits();
     if (bits == 64) {
         vmp_write_meta("bits", "64");
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -89,7 +89,7 @@
  * *************************************************************
  */
 
-static int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc)
+int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc)
 {
     PY_STACK_FRAME_T * frame;
 #ifdef RPYTHON_VMPROF
@@ -119,7 +119,7 @@
     longjmp(restore_point, SIGSEGV);
 }
 
-static int _vmprof_sample_stack(struct profbuf_s *p, PY_THREAD_STATE_T * tstate, ucontext_t * uc)
+int _vmprof_sample_stack(struct profbuf_s *p, PY_THREAD_STATE_T * tstate, ucontext_t * uc)
 {
     int depth;
     struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;

From pypy.commits at gmail.com  Mon Feb 13 10:12:14 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 13 Feb 2017 07:12:14 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: move define to vmprof.h
Message-ID: <58a1ccce.499adf0a.21fff.b254@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90089:7c63ac203256
Date: 2017-02-13 16:11 +0100
http://bitbucket.org/pypy/pypy/changeset/7c63ac203256/

Log:	move define to vmprof.h

diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
--- a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
+++ b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h
@@ -5,8 +5,6 @@
 
 #include "rvmprof.h"
 
-#define DYN_JIT_FLAG 0xbeefbeef
-
 RPY_EXTERN int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr,
                               const char * name);
 RPY_EXTERN int vmp_dyn_cancel(int ref);
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -25,6 +25,8 @@
 #define PROFILE_NATIVE '\x04'
 #define PROFILE_RPYTHON '\x08'
 
+#define DYN_JIT_FLAG 0xbeefbeef
+
 #ifdef VMPROF_UNIX
 #define VMP_SUPPORTS_NATIVE_PROFILING
 #endif

From pypy.commits at gmail.com  Mon Feb 13 11:15:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 08:15:03 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix on 32-bit
Message-ID: <58a1db87.56711c0a.91eee.db59@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90090:5ed6992ffdcd
Date: 2017-02-13 16:08 +0100
http://bitbucket.org/pypy/pypy/changeset/5ed6992ffdcd/

Log:	fix on 32-bit

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -160,8 +160,11 @@
                 w_value = self.space.newint(intmask(value))
             else:
                 w_value = self.space.newint(value)
-        elif isinstance(value, r_longlong) and -maxint-1 <= value <= maxint:
-            w_value = self.space.newint(intmask(value))
+        elif isinstance(value, r_longlong):
+            if value == r_longlong(intmask(value)):
+                w_value = self.space.newint(intmask(value))
+            else:
+                w_value = self.space.newint(value)
         elif isinstance(value, int):
             w_value = self.space.newint(value)
         elif isinstance(value, float):

From pypy.commits at gmail.com  Mon Feb 13 11:15:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 08:15:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 2601d8768d87
Message-ID: <58a1db89.c3a4df0a.6ec44.ca5e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90091:dce0ae8e9a3e
Date: 2017-02-13 17:00 +0100
http://bitbucket.org/pypy/pypy/changeset/dce0ae8e9a3e/

Log:	hg merge 2601d8768d87

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -190,7 +190,7 @@
             space.setitem(w_globals, space.newtext('__builtins__'),
                           space.builtin_modules['builtins'])
             space.setitem(w_globals, space.newtext('c_argument'),
-                          space.newbytes(c_argument))
+                          space.newint(c_argument))
             space.appexec([space.newtext(source), w_globals], """(src, glob):
                 import sys
                 stmt = compile(src, 'c callback', 'exec')
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -595,7 +595,7 @@
                         except IndexError:
                             name = '?'
                         else:
-                            name = space.text_w(w_name)
+                            name = space.identifier_w(w_name)
                     break
         self.kwd_name = name
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -480,7 +480,7 @@
         assert fields == []
 
     def __spacebind__(self, space):
-        return space.newtuple([])
+        return space.newtuple([space.newtext(field) for field in self.fields])
 
 
 class W_AST(W_Root):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -52,7 +52,7 @@
         w_dict = self.getdict(space)
         if w_dict is not None:
             try:
-                space.delitem(w_dict, space.wrap(attr))
+                space.delitem(w_dict, space.newtext(attr))
                 return True
             except OperationError as ex:
                 if not ex.match(space, space.w_KeyError):
@@ -78,7 +78,7 @@
 
     def getname(self, space):
         try:
-            return space.unicode_w(space.getattr(self, space.wrap('__name__')))
+            return space.text_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return u'?'
@@ -105,7 +105,7 @@
 
     def getrepr(self, space, info, moreinfo=u''):
         addrstring = unicode(self.getaddrstring(space))
-        return space.wrap(u"<%s at 0x%s%s>" % (info, addrstring, moreinfo))
+        return space.newunicode(u"<%s at 0x%s%s>" % (info, addrstring, moreinfo))
 
     def getslotvalue(self, index):
         raise NotImplementedError
@@ -1849,7 +1849,7 @@
         source = py.code.Source("def anonymous%s\n" % source)
         w_glob = space.newdict(module=True)
         space.exec_(str(source), w_glob, w_glob)
-        return space.getitem(w_glob, space.wrap('anonymous'))
+        return space.getitem(w_glob, space.newtext('anonymous'))
 
 
 # Table describing the regular part of the interface of object spaces,
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -89,9 +89,8 @@
             exc_typename = str(self.w_type)
             exc_value = str(w_value)
         else:
-            w = space.wrap
-            exc_typename = space.str_w(
-                space.getattr(self.w_type, w('__name__')))
+            exc_typename = space.text_w(
+                space.getattr(self.w_type, space.newtext('__name__')))
             if space.is_w(w_value, space.w_None):
                 exc_value = ""
             else:
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -7,8 +7,8 @@
 
 def app_profile_call(space, w_callable, frame, event, w_arg):
     space.call_function(w_callable,
-                        space.wrap(frame),
-                        space.wrap(event), w_arg)
+                        frame,
+                        space.newtext(event), w_arg)
 
 class ExecutionContext(object):
     """An ExecutionContext holds the state of an execution thread
@@ -312,7 +312,7 @@
                 operr.normalize_exception(space)
                 w_value = operr.get_w_value(space)
                 w_arg = space.newtuple([operr.w_type, w_value,
-                                     space.wrap(operr.get_traceback())])
+                                        operr.get_traceback()])
 
             d = frame.getorcreatedebug()
             if d.w_locals is not None:
@@ -323,7 +323,7 @@
             self.is_tracing += 1
             try:
                 try:
-                    w_result = space.call_function(w_callback, space.wrap(frame), space.wrap(event), w_arg)
+                    w_result = space.call_function(w_callback, frame, space.newtext(event), w_arg)
                     if space.is_w(w_result, space.w_None):
                         # bug-to-bug compatibility with CPython
                         # http://bugs.python.org/issue11992
@@ -601,7 +601,7 @@
         msg = ("RPython exception %s in %s<%s at 0x%s> ignored\n" % (
                    str(e), where, space.type(w_obj).name, addrstring))
         space.call_method(space.sys.get('stderr'), 'write',
-                          space.wrap(msg))
+                          space.newtext(msg))
 
 
 def make_finalizer_queue(W_Root, space):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -249,7 +249,7 @@
         func = space.allocate_instance(Function, w_subtype)
         Function.__init__(func, space, code, w_globals, defs_w, None, closure,
                           None,name)
-        return space.wrap(func)
+        return func
 
     def descr_function_call(self, __args__):
         return self.call_args(__args__)
@@ -289,14 +289,13 @@
         if isinstance(code, BuiltinCode):
             new_inst = mod.get('builtin_function')
             return space.newtuple([new_inst,
-                                   space.newtuple([space.wrap(code.identifier)])])
+                                   space.newtuple([space.newtext(code.identifier)])])
 
         new_inst = mod.get('func_new')
-        w = space.wrap
         if self.closure is None:
             w_closure = space.w_None
         else:
-            w_closure = space.newtuple([w(cell) for cell in self.closure])
+            w_closure = space.newtuple([cell for cell in self.closure])
         if self.w_doc is None:
             w_doc = space.w_None
         else:
@@ -313,10 +312,10 @@
         nt = space.newtuple
         tup_base = []
         tup_state = [
-            w(self.name),
-            w(self.qualname),
+            space.newtext(self.name),
+            space.newunicode(self.qualname),
             w_doc,
-            w(self.code),
+            self.code,
             w_func_globals,
             w_closure,
             nt(self.defs_w),
@@ -337,8 +336,8 @@
                         "Wrong arguments to function.__setstate__")
 
         self.space = space
-        self.name = space.str_w(w_name)
-        self.qualname = space.str_w(w_qualname).decode("utf-8")
+        self.name = space.text_w(w_name)
+        self.qualname = space.unicode_w(w_qualname)
         self.code = space.interp_w(Code, w_code)
         if not space.is_w(w_closure, space.w_None):
             from pypy.interpreter.nestedscope import Cell
@@ -421,7 +420,7 @@
         self.w_doc = w_doc
 
     def fget_func_name(self, space):
-        return space.wrap(self.name.decode('utf-8'))
+        return space.newtext(self.name)
 
     def fset_func_name(self, space, w_name):
         if space.isinstance_w(w_name, space.w_unicode):
@@ -431,7 +430,7 @@
                         "__name__ must be set to a string object")
 
     def fget_func_qualname(self, space):
-        return space.wrap(self.qualname)
+        return space.newunicode(self.qualname)
 
     def fset_func_qualname(self, space, w_name):
         try:
@@ -448,7 +447,7 @@
     def fget___module__(self, space):
         if self.w_module is None:
             if self.w_func_globals is not None and not space.is_w(self.w_func_globals, space.w_None):
-                self.w_module = space.call_method(self.w_func_globals, "get", space.wrap("__name__"))
+                self.w_module = space.call_method(self.w_func_globals, "get", space.newtext("__name__"))
             else:
                 self.w_module = space.w_None
         return self.w_module
@@ -460,7 +459,7 @@
         self.w_module = space.w_None
 
     def fget_func_code(self, space):
-        return space.wrap(self.code)
+        return self.code
 
     def fset_func_code(self, space, w_code):
         from pypy.interpreter.pycode import PyCode
@@ -480,7 +479,7 @@
 
     def fget_func_closure(self, space):
         if self.closure is not None:
-            w_res = space.newtuple([space.wrap(i) for i in self.closure])
+            w_res = space.newtuple([cell for cell in self.closure])
         else:
             w_res = space.w_None
         return w_res
@@ -508,7 +507,7 @@
     if w_obj is None or space.is_w(w_obj, space.w_None):
         return w_function
     else:
-        return space.wrap(Method(space, w_function, w_obj))
+        return Method(space, w_function, w_obj)
 
 
 class Method(W_Root):
@@ -527,7 +526,7 @@
             raise oefmt(space.w_TypeError, "self must not be None")
         method = space.allocate_instance(Method, w_subtype)
         Method.__init__(method, space, w_function, w_instance)
-        return space.wrap(method)
+        return method
 
     def __repr__(self):
         return u"bound method %s" % (self.w_function.getname(self.space),)
@@ -537,14 +536,14 @@
         return space.call_obj_args(self.w_function, self.w_instance, args)
 
     def descr_method_get(self, w_obj, w_cls=None):
-        return self.space.wrap(self)    # already bound
+        return self     # already bound
 
     def descr_method_call(self, __args__):
         return self.call_args(__args__)
 
     def descr_method_repr(self):
         space = self.space
-        w_name = space.findattr(self.w_function, space.wrap('__qualname__'))
+        w_name = space.findattr(self.w_function, space.newtext('__qualname__'))
         if w_name is None:
             name = self.w_function.getname(self.space)
         else:
@@ -556,14 +555,14 @@
                 name = u'?'
         objrepr = space.unicode_w(space.repr(self.w_instance))
         s = u'<bound method %s of %s>' % (name, objrepr)
-        return space.wrap(s)
+        return space.newunicode(s)
 
     def descr_method_getattribute(self, w_attr):
         space = self.space
         if space.str_w(w_attr) != '__doc__':
             try:
                 return space.call_method(space.w_object, '__getattribute__',
-                                         space.wrap(self), w_attr)
+                                         self, w_attr)
             except OperationError as e:
                 if not e.match(space, space.w_AttributeError):
                     raise
@@ -612,7 +611,7 @@
         if (isinstance(w_function, Function) and
                 isinstance(w_function.code, BuiltinCode)):
             new_inst = mod.get('builtin_method_new')
-            tup = [w_instance, space.wrap(w_function.name)]
+            tup = [w_instance, space.newtext(w_function.name)]
         else:
             new_inst = mod.get('method_new')
             tup = [self.w_function, w_instance]
@@ -678,7 +677,7 @@
     def descr_classmethod_get(self, space, w_obj, w_klass=None):
         if space.is_none(w_klass):
             w_klass = space.type(w_obj)
-        return space.wrap(Method(space, self.w_function, w_klass))
+        return Method(space, self.w_function, w_klass)
 
     def descr_classmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(ClassMethod, w_subtype)
@@ -713,10 +712,10 @@
                     "cannot create 'builtin_function' instances")
 
     def descr_function_repr(self):
-        return self.space.wrap('<built-in function %s>' % (self.name,))
+        return self.space.newtext('<built-in function %s>' % (self.name,))
 
     def descr__reduce__(self, space):
-        return space.wrap(self.qualname)
+        return space.newunicode(self.qualname)
 
 def is_builtin_code(w_func):
     from pypy.interpreter.gateway import BuiltinCode
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -731,7 +731,7 @@
         mod = space.interp_w(MixedModule, w_mod)
         builtin_code = mod.get('builtin_code')
         return space.newtuple([builtin_code,
-                               space.newtuple([space.wrap(self.identifier)])])
+                               space.newtuple([space.newtext(self.identifier)])])
 
     @staticmethod
     def find(space, identifier):
@@ -742,7 +742,7 @@
         return self.sig
 
     def getdocstring(self, space):
-        return space.wrap(self.docstring)
+        return space.newtext(self.docstring)
 
     def funcrun(self, func, args):
         return BuiltinCode.funcrun_obj(self, func, None, args)
@@ -1135,11 +1135,11 @@
 
     def buildmodule(self, space, name='applevel'):
         from pypy.interpreter.module import Module
-        return Module(space, space.wrap(name), self.getwdict(space))
+        return Module(space, space.newtext(name), self.getwdict(space))
 
     def wget(self, space, name):
         w_globals = self.getwdict(space)
-        return space.getitem(w_globals, space.wrap(name))
+        return space.getitem(w_globals, space.newtext(name))
 
     def interphook(self, name):
         "NOT_RPYTHON"
@@ -1189,7 +1189,7 @@
 def build_applevel_dict(self, space):
     "NOT_RPYTHON"
     w_glob = space.newdict(module=True)
-    space.setitem(w_glob, space.wrap('__name__'), space.wrap(self.modname))
+    space.setitem(w_glob, space.newtext('__name__'), space.newtext(self.modname))
     space.exec_(self.source, w_glob, w_glob,
                 hidden_applevel=self.hidden_applevel,
                 filename=self.filename)
diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -93,13 +93,13 @@
 
         mainmodule = main.ensure__main__(space)
         self.w_globals = mainmodule.w_dict
-        space.setitem(self.w_globals, space.wrap('__builtins__'), space.builtin)
+        space.setitem(self.w_globals, space.newtext('__builtins__'), space.builtin)
         if completer:
             self.enable_command_line_completer()
 
         # forbidden:
         #space.exec_("__pytrace__ = 0", self.w_globals, self.w_globals)
-        space.setitem(self.w_globals, space.wrap('__pytrace__'),space.newint(0))
+        space.setitem(self.w_globals, space.newtext('__pytrace__'),space.newint(0))
         self.tracelevel = 0
         self.console_locals = {}
 
@@ -159,7 +159,7 @@
             for name in local:
                 if name.startswith('w_'):
                     self.space.setitem(self.w_globals,
-                                       self.space.wrap(name[2:]),
+                                       self.space.newtext(name[2:]),
                                        local[name])
             print '*** Leaving interpreter-level console ***'
             raise
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -4,7 +4,7 @@
 
 
 def ensure__main__(space):
-    w_main = space.wrap('__main__')
+    w_main = space.newtext('__main__')
     w_modules = space.sys.get('modules')
     try:
         return space.getitem(w_modules, w_main)
@@ -17,10 +17,9 @@
 
 
 def compilecode(space, source, filename, cmd='exec'):
-    w = space.wrap
     w_code = space.builtin.call(
         'compile', space.newbytes(source), space.wrap_fsdecoded(filename),
-        w(cmd), w(0), w(0))
+        space.wrap(cmd), space.newint(0), space.newint(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
 
@@ -36,16 +35,14 @@
             from pypy.objspace.std import StdObjSpace
             space = StdObjSpace()
 
-        w = space.wrap
-
         pycode = compilecode(space, source, filename or '<string>', cmd)
 
         mainmodule = ensure__main__(space)
         w_globals = mainmodule.w_dict
 
-        space.setitem(w_globals, w('__builtins__'), space.builtin)
+        space.setitem(w_globals, space.newtext('__builtins__'), space.builtin)
         if filename is not None:
-            space.setitem(w_globals, w('__file__'),
+            space.setitem(w_globals, space.newtext('__file__'),
                           space.wrap_fsdecoded(filename))
 
         retval = pycode.exec_code(space, w_globals, w_globals)
@@ -85,16 +82,15 @@
     if space is None:
         from pypy.objspace.std import StdObjSpace
         space = StdObjSpace()
-    w = space.wrap
     argv = [module_name]
     if args is not None:
         argv.extend(args)
-    space.setitem(space.sys.w_dict, w('argv'), w(argv))
+    space.setitem(space.sys.w_dict, space.newtext('argv'), space.wrap(argv))
     w_import = space.builtin.get('__import__')
-    runpy = space.call_function(w_import, w('runpy'))
-    w_run_module = space.getitem(runpy.w_dict, w('run_module'))
-    return space.call_function(w_run_module, w(module_name), space.w_None,
-                               w('__main__'), space.w_True)
+    runpy = space.call_function(w_import, space.wrap('runpy'))
+    w_run_module = space.getitem(runpy.w_dict, space.wrap('run_module'))
+    return space.call_function(w_run_module, space.wrap(module_name), space.w_None,
+                               space.wrap('__main__'), space.w_True)
 
 
 def run_toplevel(space, f, verbose=False):
@@ -121,7 +117,7 @@
             # exit if we catch a w_SystemExit
             if operationerr.match(space, space.w_SystemExit):
                 w_exitcode = space.getattr(w_value,
-                                           space.wrap('code'))
+                                           space.newtext('code'))
                 if space.is_w(w_exitcode, space.w_None):
                     exitcode = 0
                 else:
@@ -135,9 +131,9 @@
                 raise SystemExit(exitcode)
 
             # set the sys.last_xxx attributes
-            space.setitem(space.sys.w_dict, space.wrap('last_type'), w_type)
-            space.setitem(space.sys.w_dict, space.wrap('last_value'), w_value)
-            space.setitem(space.sys.w_dict, space.wrap('last_traceback'),
+            space.setitem(space.sys.w_dict, space.newtext('last_type'), w_type)
+            space.setitem(space.sys.w_dict, space.newtext('last_value'), w_value)
+            space.setitem(space.sys.w_dict, space.newtext('last_traceback'),
                           w_traceback)
 
             # call sys.excepthook if present
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -34,7 +34,7 @@
             for sub_name, module_cls in self.submodules.iteritems():
                 if module_cls.submodule_name is None:
                     module_cls.submodule_name = sub_name
-                module_name = space.wrap("%s.%s" % (name, sub_name))
+                module_name = space.newtext("%s.%s" % (name, sub_name))
                 m = module_cls(space, module_name)
                 m.install()
                 self.submodules_w.append(m)
@@ -50,7 +50,7 @@
 
         for w_submodule in self.submodules_w:
             name = space.str0_w(w_submodule.w_name)
-            space.setitem(self.w_dict, space.wrap(name.split(".")[-1]), w_submodule)
+            space.setitem(self.w_dict, space.newtext(name.split(".")[-1]), w_submodule)
             space.getbuiltinmodule(name)
 
         if self.w_initialdict is None:
@@ -77,7 +77,7 @@
         space = self.space
         w_value = self.getdictvalue(space, name)
         if w_value is None:
-            raise OperationError(space.w_AttributeError, space.wrap(name))
+            raise OperationError(space.w_AttributeError, space.newtext(name))
         return w_value
 
     def call(self, name, *args_w):
@@ -186,7 +186,7 @@
         loader = getinterpevalloader(pkgroot, spec)
         space = self.space
         w_obj = loader(space)
-        space.setattr(space.wrap(self), space.wrap(name), w_obj)
+        space.setattr(self, space.newtext(name), w_obj)
 
     @classmethod
     def get__doc__(cls, space):
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -31,15 +31,14 @@
         statically inside the executable."""
         try:
             space = self.space
-            space.delitem(self.w_dict, space.wrap('__file__'))
+            space.delitem(self.w_dict, space.newtext('__file__'))
         except OperationError:
             pass
 
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
-        w_mod = self.space.wrap(self)
         modulename = self.space.str0_w(self.w_name)
-        self.space.builtin_modules[modulename] = w_mod
+        self.space.builtin_modules[modulename] = self
 
     def setup_after_space_initialization(self):
         """NOT_RPYTHON: to allow built-in modules to do some more setup
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -60,7 +60,7 @@
                                space.newtuple(tup)])
 
     def descr__setstate__(self, space, w_state):
-        self.w_value = space.getitem(w_state, space.wrap(0))
+        self.w_value = space.getitem(w_state, space.newint(0))
 
     def __repr__(self):
         """ representation for debugging purposes """
@@ -78,7 +78,7 @@
             content = "%s object at 0x%s" % (space.type(self.w_value).name,
                                              self.w_value.getaddrstring(space))
         s = "<cell at 0x%s: %s>" % (self.getaddrstring(space), content)
-        return space.wrap(s.decode('utf-8'))
+        return space.newtext(s)
 
     def descr__cell_contents(self, space):
         try:
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -367,7 +367,7 @@
         for name in self.co_varnames:  result ^= compute_hash(name)
         for name in self.co_freevars:  result ^= compute_hash(name)
         for name in self.co_cellvars:  result ^= compute_hash(name)
-        w_result = space.wrap(intmask(result))
+        w_result = space.newint(intmask(result))
         for w_name in self.co_names_w:
             w_result = space.xor(w_result, space.hash(w_name))
         for w_const in self.co_consts_w:
@@ -409,31 +409,30 @@
         code = space.allocate_instance(PyCode, w_subtype)
         PyCode.__init__(code, space, argcount, kwonlyargcount, nlocals, stacksize, flags, codestring, consts_w[:], names,
                       varnames, filename, name, firstlineno, lnotab, freevars, cellvars, magic=magic)
-        return space.wrap(code)
+        return code
 
     def descr__reduce__(self, space):
         from pypy.interpreter.mixedmodule import MixedModule
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
         new_inst = mod.get('code_new')
-        w        = space.wrap
         tup      = [
-            w(self.co_argcount),
-            w(self.co_kwonlyargcount),
-            w(self.co_nlocals),
-            w(self.co_stacksize),
-            w(self.co_flags),
+            space.newint(self.co_argcount),
+            space.newint(self.co_kwonlyargcount),
+            space.newint(self.co_nlocals),
+            space.newint(self.co_stacksize),
+            space.newint(self.co_flags),
             space.newbytes(self.co_code),
             space.newtuple(self.co_consts_w),
             space.newtuple(self.co_names_w),
-            space.newtuple([w(v) for v in self.co_varnames]),
-            w(self.co_filename),
-            w(self.co_name),
-            w(self.co_firstlineno),
+            space.newtuple([space.newtext(v) for v in self.co_varnames]),
+            space.newtext(self.co_filename),
+            space.newtext(self.co_name),
+            space.newint(self.co_firstlineno),
             space.newbytes(self.co_lnotab),
-            space.newtuple([w(v) for v in self.co_freevars]),
-            space.newtuple([w(v) for v in self.co_cellvars]),
-            w(self.magic),
+            space.newtuple([space.newtext(v) for v in self.co_freevars]),
+            space.newtuple([space.newtext(v) for v in self.co_cellvars]),
+            space.newint(self.magic),
         ]
         return space.newtuple([new_inst, space.newtuple(tup)])
 
@@ -452,6 +451,6 @@
         # co_name should be an identifier
         name = self.co_name.decode('utf-8')
         fn = space.fsdecode_w(space.newbytes(self.co_filename))
-        return space.wrap(u'<code object %s at 0x%s, file "%s", line %d>' % (
+        return space.newunicode(u'<code object %s at 0x%s, file "%s", line %d>' % (
             name, unicode(self.getaddrstring(space)), fn,
             -1 if self.co_firstlineno == 0 else self.co_firstlineno))
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -293,7 +293,7 @@
             self.f_generator_wref = rweakref.ref(gen)
         else:
             self.f_generator_nowref = gen
-        w_gen = space.wrap(gen)
+        w_gen = gen
 
         if w_wrapper is not None:
             if ec.in_coroutine_wrapper:
@@ -535,7 +535,7 @@
             if w_value is not None:
                 self.space.setitem_str(d.w_locals, name, w_value)
             else:
-                w_name = self.space.wrap(name.decode('utf-8'))
+                w_name = self.space.newtext(name)
                 try:
                     self.space.delitem(d.w_locals, w_name)
                 except OperationError as e:
@@ -613,7 +613,7 @@
         return None
 
     def fget_code(self, space):
-        return space.wrap(self.getcode())
+        return self.getcode()
 
     def fget_getdictscope(self, space):
         return self.getdictscope()
@@ -628,9 +628,9 @@
     def fget_f_lineno(self, space):
         "Returns the line number of the instruction currently being executed."
         if self.get_w_f_trace() is None:
-            return space.wrap(self.get_last_lineno())
+            return space.newint(self.get_last_lineno())
         else:
-            return space.wrap(self.getorcreatedebug().f_lineno)
+            return space.newint(self.getorcreatedebug().f_lineno)
 
     def fset_f_lineno(self, space, w_new_lineno):
         "Change the line number of the instruction currently being executed."
@@ -774,10 +774,10 @@
 
     def fget_f_back(self, space):
         f_back = ExecutionContext.getnextframe_nohidden(self)
-        return self.space.wrap(f_back)
+        return f_back
 
     def fget_f_lasti(self, space):
-        return self.space.wrap(self.last_instr)
+        return self.space.newint(self.last_instr)
 
     def fget_f_trace(self, space):
         return self.get_w_f_trace()
@@ -795,7 +795,7 @@
 
     def fget_f_restricted(self, space):
         if space.config.objspace.honor__builtins__:
-            return space.wrap(self.builtin is not space.builtin)
+            return space.newbool(self.builtin is not space.builtin)
         return space.w_False
 
     def get_generator(self):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -86,7 +86,7 @@
             rstackovf.check_stack_overflow()
             next_instr = self.handle_asynchronous_error(ec,
                 self.space.w_RecursionError,
-                self.space.wrap("maximum recursion depth exceeded"))
+                self.space.newtext("maximum recursion depth exceeded"))
         return next_instr
 
     def handle_asynchronous_error(self, ec, w_type, w_value=None):
@@ -557,18 +557,17 @@
     def raise_exc_unbound(self, varindex):
         varname = self.getfreevarname(varindex)
         if self.iscellvar(varindex):
-            message = "local variable '%s' referenced before assignment"%varname
-            w_exc_type = self.space.w_UnboundLocalError
+            raise oefmt(self.space.w_UnboundLocalError,
+                        "local variable '%s' referenced before assignment",
+                        varname)
         else:
-            message = ("free variable '%s' referenced before assignment"
-                       " in enclosing scope"%varname)
-            w_exc_type = self.space.w_NameError
-        raise OperationError(w_exc_type, self.space.wrap(message))
+            raise oefmt(self.space.w_NameError,
+                        "free variable '%s' referenced before assignment"
+                        " in enclosing scope", varname)
 
     def LOAD_CLOSURE(self, varindex, next_instr):
         # nested scopes: access the cell object
-        cell = self._getcell(varindex)
-        w_value = self.space.wrap(cell)
+        w_value = self._getcell(varindex)
         self.pushvalue(w_value)
 
     def POP_TOP(self, oparg, next_instr):
@@ -1227,7 +1226,7 @@
         if isinstance(w_unroller, SApplicationException):
             # app-level exception
             operr = w_unroller.operr
-            w_traceback = self.space.wrap(operr.get_traceback())
+            w_traceback = operr.get_traceback()
             w_res = self.call_contextmanager_exit_function(
                 w_exitfunc,
                 operr.w_type,
@@ -1341,7 +1340,7 @@
         fn = function.Function(space, codeobj, self.get_w_globals(),
                                defaultarguments,
                                kw_defs_w, freevars, w_ann, qualname=qualname)
-        self.pushvalue(space.wrap(fn))
+        self.pushvalue(fn)
 
     def MAKE_FUNCTION(self, oparg, next_instr):
         return self._make_function(oparg)
@@ -1830,7 +1829,7 @@
         assert isinstance(unroller, SApplicationException)
         operationerr = unroller.operr
         operationerr.normalize_exception(frame.space)
-        frame.pushvalue(frame.space.wrap(unroller))
+        frame.pushvalue(unroller)
         frame.pushvalue(operationerr.get_w_value(frame.space))
         frame.pushvalue(operationerr.w_type)
         # set the current value of sys_exc_info to operationerr,
@@ -1855,7 +1854,7 @@
         if isinstance(unroller, SApplicationException):
             operationerr = unroller.operr
             operationerr.normalize_exception(frame.space)
-        frame.pushvalue(frame.space.wrap(unroller))
+        frame.pushvalue(unroller)
         # set the current value of sys_exc_info to operationerr,
         # saving the old value in a custom type of FrameBlock
         frame.save_and_change_sys_exc_info(operationerr)
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -28,15 +28,15 @@
             if len(self.text) != offset:
                 text, _ = str_decode_utf_8(self.text, len(self.text),
                                            'replace')
-            w_text = space.wrap(text)
+            w_text = space.newtext(text)
         if self.filename is not None:
             w_filename = space.wrap_fsdecoded(self.filename)
         return space.newtuple([space.wrap(self.msg),
                                space.newtuple([w_filename,
-                                               space.wrap(self.lineno),
-                                               space.wrap(offset),
+                                               space.newint(self.lineno),
+                                               space.newint(offset),
                                                w_text,
-                                               space.wrap(self.lastlineno)])])
+                                               space.newint(self.lastlineno)])])
 
     def __str__(self):
         return "%s at pos (%d, %d) in %r" % (self.__class__.__name__,
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -24,20 +24,19 @@
         return offset2lineno(self.frame.pycode, self.lasti)
 
     def descr_tb_lineno(self, space):
-        return space.wrap(self.get_lineno())
+        return space.newint(self.get_lineno())
 
     def descr__reduce__(self, space):
         from pypy.interpreter.mixedmodule import MixedModule
         w_mod = space.getbuiltinmodule('_pickle_support')
         mod = space.interp_w(MixedModule, w_mod)
         new_inst = mod.get('traceback_new')
-        w = space.wrap
 
         tup_base = []
         tup_state = [
-            w(self.frame),
-            w(self.lasti),
-            w(self.next),
+            self.frame,
+            space.newint(self.lasti),
+            self.next,
         ]
         nt = space.newtuple
         return nt([new_inst, nt(tup_base), nt(tup_state)])
@@ -65,5 +64,5 @@
 
 def check_traceback(space, w_tb, msg):
     if w_tb is None or not space.isinstance_w(w_tb, space.gettypeobject(PyTraceback.typedef)):
-        raise OperationError(space.w_TypeError, space.wrap(msg))
+        raise OperationError(space.w_TypeError, space.newtext(msg))
     return w_tb
diff --git a/pypy/interpreter/special.py b/pypy/interpreter/special.py
--- a/pypy/interpreter/special.py
+++ b/pypy/interpreter/special.py
@@ -8,7 +8,7 @@
         return space.w_Ellipsis
 
     def descr__repr__(self, space):
-        return space.wrap('Ellipsis')
+        return space.newtext('Ellipsis')
 
     descr__reduce__ = descr__repr__
 
@@ -20,6 +20,6 @@
         return space.w_NotImplemented
 
     def descr__repr__(self, space):
-        return space.wrap('NotImplemented')
+        return space.newtext('NotImplemented')
 
     descr__reduce__ = descr__repr__
diff --git a/pypy/interpreter/streamutil.py b/pypy/interpreter/streamutil.py
--- a/pypy/interpreter/streamutil.py
+++ b/pypy/interpreter/streamutil.py
@@ -4,7 +4,7 @@
 def wrap_streamerror(space, e, w_filename=None):
     if isinstance(e, StreamError):
         return OperationError(space.w_ValueError,
-                              space.wrap(e.message))
+                              space.newtext(e.message))
     elif isinstance(e, OSError):
         return wrap_oserror_as_ioerror(space, e, w_filename)
     else:
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -94,6 +94,7 @@
 
     def wrap(self, obj):
         return obj
+    newtext = wrap
 
     def str_w(self, s):
         return str(s)

From pypy.commits at gmail.com  Mon Feb 13 11:15:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 08:15:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: simplify
Message-ID: <58a1db8b.44641c0a.692b4.dd96@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90092:f922549d71f7
Date: 2017-02-13 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/f922549d71f7/

Log:	simplify

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -557,13 +557,13 @@
     def raise_exc_unbound(self, varindex):
         varname = self.getfreevarname(varindex)
         if self.iscellvar(varindex):
-            message = "local variable '%s' referenced before assignment"%varname
-            w_exc_type = self.space.w_UnboundLocalError
+            raise oefmt(self.space.w_UnboundLocalError,
+                        "local variable '%s' referenced before assignment",
+                        varname)
         else:
-            message = ("free variable '%s' referenced before assignment"
-                       " in enclosing scope"%varname)
-            w_exc_type = self.space.w_NameError
-        raise OperationError(w_exc_type, self.space.wrap(message))
+            raise oefmt(self.space.w_NameError,
+                        "free variable '%s' referenced before assignment"
+                        " in enclosing scope", varname)
 
     def LOAD_CLOSURE(self, varindex, next_instr):
         # nested scopes: access the cell object

From pypy.commits at gmail.com  Mon Feb 13 11:15:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 08:15:09 -0800 (PST)
Subject: [pypy-commit] pypy default: simplify
Message-ID: <58a1db8d.d18bdf0a.4bf87.c2ee@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90093:21fe41563985
Date: 2017-02-13 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/21fe41563985/

Log:	simplify

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1010,8 +1010,7 @@
 
         w_import = self.get_builtin().getdictvalue(space, '__import__')
         if w_import is None:
-            raise OperationError(space.w_ImportError,
-                                 space.wrap("__import__ not found"))
+            raise oefmt(space.w_ImportError, "__import__ not found")
         d = self.getdebug()
         if d is None:
             w_locals = None

From pypy.commits at gmail.com  Mon Feb 13 11:15:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 08:15:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: some post-merge fixes
Message-ID: <58a1db8f.977d1c0a.23f4a.d9e1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90094:72f4854167dd
Date: 2017-02-13 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/72f4854167dd/

Log:	some post-merge fixes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -78,7 +78,7 @@
 
     def getname(self, space):
         try:
-            return space.text_w(space.getattr(self, space.newtext('__name__')))
+            return space.unicode_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return u'?'
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -30,7 +30,7 @@
         Module.install(self)
         if hasattr(self, "submodules"):
             space = self.space
-            name = space.unwrap(self.w_name)
+            name = space.text_w(self.w_name)
             for sub_name, module_cls in self.submodules.iteritems():
                 if module_cls.submodule_name is None:
                     module_cls.submodule_name = sub_name
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -28,7 +28,7 @@
             if len(self.text) != offset:
                 text, _ = str_decode_utf_8(self.text, len(self.text),
                                            'replace')
-            w_text = space.newtext(text)
+            w_text = space.newunicode(text)
         if self.filename is not None:
             w_filename = space.wrap_fsdecoded(self.filename)
         return space.newtuple([space.wrap(self.msg),
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -98,6 +98,7 @@
 
     def str_w(self, s):
         return str(s)
+    text_w = str_w
 
     def unicode_w(self, s):
         return unicode(s)
@@ -352,6 +353,7 @@
                 raise OperationError(ValueError, None)
             return str(w)
         space.unicode_w = unicode_w
+        space.text_w = unicode_w
         excinfo = py.test.raises(OperationError, Arguments, space, [],
                                  ["a"], [1], w_starstararg={None: 1})
         assert excinfo.value.w_type is TypeError
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -779,7 +779,7 @@
 
 def descr_get__name__(space, w_type):
     w_type = _check(space, w_type)
-    return space.newtext(w_type.getname(space))
+    return space.newunicode(w_type.getname(space))
 
 def descr_set__name__(space, w_type, w_value):
     w_type = _check(space, w_type)

From pypy.commits at gmail.com  Mon Feb 13 11:18:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 08:18:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix
Message-ID: <58a1dc38.496f1c0a.7fc9.dea5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90095:2c2584ce56cf
Date: 2017-02-13 17:17 +0100
http://bitbucket.org/pypy/pypy/changeset/2c2584ce56cf/

Log:	fix

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -152,7 +152,7 @@
         storage = strategy.get_empty_storage()
         d_new = strategy.unerase(storage)
         for i in range(len(keys)):
-            d_new[self.decodekey_str(keys[i])] = values_w[i]
+            d_new[strategy.decodekey_str(keys[i])] = values_w[i]
         w_dict.set_strategy(strategy)
         w_dict.dstorage = storage
 

From pypy.commits at gmail.com  Mon Feb 13 11:40:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 08:40:59 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58a1e19b.50a4df0a.835d.d49e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r858:69f598252754
Date: 2017-02-13 17:40 +0100
http://bitbucket.org/pypy/pypy.org/changeset/69f598252754/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66596 of $105000 (63.4%)
+   $66620 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2422 left
+    we have $2444 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Mon Feb 13 12:33:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 09:33:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge c945ce239b8c
Message-ID: <58a1edd1.d8301c0a.4d323.0d97@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90096:f298674b1732
Date: 2017-02-13 18:32 +0100
http://bitbucket.org/pypy/pypy/changeset/f298674b1732/

Log:	hg merge c945ce239b8c

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -269,8 +269,8 @@
             else:
                 w_key = space.newtuple([obj, space.w_float])
         elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
+            w_real = space.getattr(obj, space.newtext("real"))
+            w_imag = space.getattr(obj, space.newtext("imag"))
             real = space.float_w(w_real)
             imag = space.float_w(w_imag)
             real_negzero = (real == 0.0 and
@@ -369,7 +369,7 @@
         space = self.space
         consts_w = [space.w_None] * space.len_w(w_consts)
         w_iter = space.iter(w_consts)
-        first = space.wrap(0)
+        first = space.newint(0)
         while True:
             try:
                 w_key = space.next(w_iter)
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1168,8 +1168,8 @@
                 raw = "0" + raw
         if negative:
             raw = "-" + raw
-        w_num_str = self.space.wrap(raw)
-        w_base = self.space.wrap(base)
+        w_num_str = self.space.newtext(raw)
+        w_base = self.space.newint(base)
         if raw[-1] in "jJ":
             tp = self.space.w_complex
             return self.space.call_function(tp, w_num_str)
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -313,7 +313,7 @@
 
     def _make_function(self, code, num_defaults=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
-        w_qualname = self.space.wrap((qualname or code.co_name).decode('utf-8'))
+        w_qualname = self.space.newtext(qualname or code.co_name)
         if code.co_freevars:
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
@@ -339,8 +339,8 @@
             if default:
                 kwonly = args.kwonlyargs[i]
                 assert isinstance(kwonly, ast.arg)
-                mangled = self.scope.mangle(kwonly.arg).decode('utf-8')
-                self.load_const(self.space.wrap(mangled))
+                mangled = self.scope.mangle(kwonly.arg)
+                self.load_const(self.space.newtext(mangled))
                 default.walkabout(self)
                 defaults += 1
         return defaults
@@ -375,8 +375,7 @@
         if l:
             if l > 65534:
                 self.error("too many annotations", func)
-            w_tup = space.newtuple([space.wrap(name.decode('utf-8'))
-                                    for name in names])
+            w_tup = space.newtuple([space.newtext(name) for name in names])
             self.load_const(w_tup)
             l += 1
         return l
@@ -438,7 +437,7 @@
         # 3. load a function (or closure) made from the code object
         self._make_function(code, qualname=qualname)
         # 4. load class name
-        self.load_const(self.space.wrap(cls.name.decode('utf-8')))
+        self.load_const(self.space.newtext(cls.name))
         # 5. generate the rest of the code for the call
         self._make_call(2, cls.bases, cls.keywords)
         # 6. apply decorators
@@ -808,7 +807,7 @@
         for alias in imp.names:
             assert isinstance(alias, ast.alias)
             level = 0
-            self.load_const(self.space.wrap(level))
+            self.load_const(self.space.newint(level))
             self.load_const(self.space.w_None)
             self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name)
             # If there's no asname then we store the root module.  If there is
@@ -846,12 +845,12 @@
                         self.error("not a chance", imp)
                     self.error("future feature %s is not defined" %
                                (alias.name,), imp)
-        self.load_const(space.wrap(imp.level))
+        self.load_const(space.newint(imp.level))
         names_w = [None]*len(imp.names)
         for i in range(len(imp.names)):
             alias = imp.names[i]
             assert isinstance(alias, ast.alias)
-            names_w[i] = space.wrap(alias.name.decode('utf-8'))
+            names_w[i] = space.newtext(alias.name)
         self.load_const(space.newtuple(names_w))
         if imp.module:
             mod_name = imp.module
@@ -1250,7 +1249,7 @@
 
     def visit_keyword(self, keyword):
         if keyword.arg is not None:
-            self.load_const(self.space.wrap(keyword.arg.decode('utf-8')))
+            self.load_const(self.space.newtext(keyword.arg))
         keyword.value.walkabout(self)
 
     def _make_call(self, n, # args already pushed
@@ -1308,7 +1307,7 @@
                     nsubkwargs += 1
                 elif nsubkwargs:
                     # A keyword argument and we already have a dict.
-                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
+                    self.load_const(self.space.newtext(kw.arg))
                     kw.value.walkabout(self)
                     nseen += 1
                 else:
@@ -1651,7 +1650,7 @@
         # ... and store it as __module__
         self.name_op("__module__", ast.Store)
         # store the qualname
-        w_qualname = self.space.wrap(self.qualname.decode("utf-8"))
+        w_qualname = self.space.newtext(self.qualname)
         self.load_const(w_qualname)
         self.name_op("__qualname__", ast.Store)
         # compile the body proper
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -131,7 +131,7 @@
     return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
-    return space.wrap(not space.is_true(operand))
+    return space.newbool(not space.is_true(operand))
 
 
 binary_folders = {
@@ -222,7 +222,7 @@
                         break
                 else:
                     raise AssertionError("unknown unary operation")
-                w_minint = self.space.wrap(-sys.maxint - 1)
+                w_minint = self.space.newint(-sys.maxint - 1)
                 # This makes sure the result is an integer.
                 if self.space.eq_w(w_minint, w_const):
                     w_const = w_minint
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,6 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import dont_inline
 from rpython.rlib import rstack, rstackovf
 
 from pypy.interpreter import debug
@@ -16,7 +17,7 @@
 RECORD_INTERPLEVEL_TRACEBACK = True
 
 def strerror(errno):
-    """Translate an error code to a message string."""
+    """Translate an error code to a unicode message string."""
     from pypy.module._codecs.locale import str_decode_locale_surrogateescape
     return str_decode_locale_surrogateescape(os.strerror(errno))
 
@@ -269,7 +270,7 @@
                 pass
             w_t = self.w_type
             w_v = self.get_w_value(space)
-            w_tb = space.wrap(self.get_traceback())
+            w_tb = self.get_traceback()
             if where or objrepr:
                 if with_traceback:
                     first_line = 'From %s%s:\n' % (where, objrepr)
@@ -283,8 +284,8 @@
                 # but from `raise ValueError('foo')` you get
                 # "Exception ValueError: ValueError('foo',) in x ignored"
                 first_line = ''
-            space.appexec([space.wrap(first_line),
-                           space.wrap(extra_line),
+            space.appexec([space.newtext(first_line),
+                           space.newtext(extra_line),
                            w_t, w_v, w_tb],
             """(first_line, extra_line, t, v, tb):
                 import sys
@@ -301,7 +302,7 @@
         w_value = self._w_value
         if w_value is None:
             value = self._compute_value(space)
-            self._w_value = w_value = space.wrap(value)
+            self._w_value = w_value = space.newunicode(value)
         return w_value
 
     def _compute_value(self, space):
@@ -532,7 +533,7 @@
 
 @specialize.arg(1)
 def oefmt(w_type, valuefmt, *args):
-    """Equivalent to OperationError(w_type, space.wrap(valuefmt % args)).
+    """Equivalent to OperationError(w_type, space.newtext(valuefmt % args)).
     More efficient in the (common) case where the value is not actually
     needed. Note that in the py3k branch the exception message will
     always be unicode.
@@ -578,11 +579,11 @@
             msg = 'Windows Error %d' % winerror
         exc = space.w_WindowsError
         if w_filename is not None:
-            w_error = space.call_function(exc, space.wrap(winerror),
-                                          space.wrap(msg), w_filename)
+            w_error = space.call_function(exc, space.newint(winerror),
+                                          space.newtext(msg), w_filename)
         else:
-            w_error = space.call_function(exc, space.wrap(winerror),
-                                          space.wrap(msg))
+            w_error = space.call_function(exc, space.newint(winerror),
+                                          space.newtext(msg))
         return OperationError(exc, w_error)
 
 @specialize.arg(3, 6)
@@ -617,6 +618,7 @@
         assert operror is not None   # tell the annotator we don't return None
         return operror
 
+ at dont_inline
 def _wrap_oserror2_impl(space, e, w_filename, w_filename2, w_exc, eintr_retry):
     # move the common logic in its own function, instead of having it
     # duplicated 4 times in all 4 specialized versions of wrap_oserror2()
@@ -633,43 +635,43 @@
         msg = u'error %d' % errno
     if w_filename is not None:
         if w_filename2 is not None:
-            w_error = space.call_function(w_exc, space.wrap(errno),
-                                          space.wrap(msg), w_filename,
+            w_error = space.call_function(w_exc, space.newint(errno),
+                                          space.newunicode(msg), w_filename,
                                           space.w_None, w_filename2)
         else:
-            w_error = space.call_function(w_exc, space.wrap(errno),
-                                          space.wrap(msg), w_filename)
+            w_error = space.call_function(w_exc, space.newint(errno),
+                                          space.newunicode(msg), w_filename)
     else:
-        w_error = space.call_function(w_exc, space.wrap(errno),
-                                      space.wrap(msg))
+        w_error = space.call_function(w_exc, space.newint(errno),
+                                      space.newunicode(msg))
     operror = OperationError(w_exc, w_error)
     if eintr_retry:
         raise operror
     return operror
-_wrap_oserror2_impl._dont_inline_ = True
 
 @specialize.arg(3, 6)
+ at dont_inline
 def wrap_oserror(space, e, filename=None, exception_name='w_OSError',
                  w_exception_class=None, filename2=None, eintr_retry=False):
     w_filename = None
     w_filename2 = None
     if filename is not None:
-        w_filename = space.wrap(filename)
+        w_filename = space.wrap_fsdecoded(filename)
         if filename2 is not None:
-            w_filename2 = space.wrap(filename2)
+            w_filename2 = space.wrap_fsdecoded(filename2)
     return wrap_oserror2(space, e, w_filename,
                          exception_name=exception_name,
                          w_exception_class=w_exception_class,
                          w_filename2=w_filename2,
                          eintr_retry=eintr_retry)
-wrap_oserror._dont_inline_ = True
 
 def exception_from_saved_errno(space, w_type):
     from rpython.rlib.rposix import get_saved_errno
 
     errno = get_saved_errno()
     msg = strerror(errno)
-    w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg))
+    w_error = space.call_function(w_type, space.newint(errno),
+                                  space.newunicode(msg))
     return OperationError(w_type, w_error)
 
 def new_exception_class(space, name, w_bases=None, w_dict=None):
@@ -690,9 +692,9 @@
     if w_dict is None:
         w_dict = space.newdict()
     w_exc = space.call_function(
-        space.w_type, space.wrap(name), w_bases, w_dict)
+        space.w_type, space.newtext(name), w_bases, w_dict)
     if module:
-        space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
+        space.setattr(w_exc, space.newtext("__module__"), space.newtext(module))
     return w_exc
 
 def new_import_error(space, w_msg, w_name, w_path):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -42,7 +42,7 @@
 
     def descr__repr__(self, space):
         addrstring = self.getaddrstring(space)
-        return space.wrap(u"<%s object %s at 0x%s>" %
+        return space.newunicode(u"<%s object %s at 0x%s>" %
                           (unicode(self.KIND),
                            self.get_qualname(),
                            unicode(addrstring)))
@@ -188,7 +188,7 @@
             frame._report_stopiteration_sometimes(w_yf, e)
             try:
                 w_stop_value = space.getattr(e.get_w_value(space),
-                                             space.wrap("value"))
+                                             space.newtext("value"))
             except OperationError as e:
                 if not e.match(space, space.w_AttributeError):
                     raise
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -126,7 +126,7 @@
                     func._builtinversion_ = bltin
                     bltin.name = name
                     bltin.qualname = bltin.name.decode('utf-8')
-                w_value = space.wrap(bltin)
+                w_value = bltin
             space.setitem(self.w_dict, w_name, w_value)
             return w_value
 
@@ -190,7 +190,7 @@
 
     @classmethod
     def get__doc__(cls, space):
-        return space.wrap(cls.__doc__)
+        return space.newtext_or_none(cls.__doc__)
 
 
 def getinterpevalloader(pkgroot, spec):
@@ -218,7 +218,7 @@
             else:
                 #print spec, "->", value
                 if hasattr(value, 'func_code'):  # semi-evil
-                    return space.wrap(gateway.interp2app(value))
+                    return gateway.interp2app(value).get_function(space)
 
                 try:
                     is_type = issubclass(value, W_Root)  # pseudo-evil
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -73,7 +73,7 @@
     def descr_module__new__(space, w_subtype, __args__):
         module = space.allocate_instance(Module, w_subtype)
         Module.__init__(module, space, None)
-        return space.wrap(module)
+        return module
 
     def descr_module__init__(self, w_name, w_doc=None):
         space = self.space
@@ -86,7 +86,7 @@
         init_extra_module_attrs(space, space.wrap(self))
 
     def descr__reduce__(self, space):
-        w_name = space.finditem(self.w_dict, space.wrap('__name__'))
+        w_name = space.finditem(self.w_dict, space.newtext('__name__'))
         if (w_name is None or
             not space.isinstance_w(w_name, space.w_unicode)):
             # maybe raise exception here (XXX this path is untested)
@@ -110,7 +110,7 @@
                 w_name,
                 space.w_None,
                 space.w_None,
-                space.newtuple([space.wrap('')])
+                space.newtuple([space.newtext('')])
             ])
         ]
 
@@ -127,7 +127,7 @@
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
                 raise
-            w_name = space.finditem(self.w_dict, space.wrap('__name__'))
+            w_name = space.finditem(self.w_dict, space.newtext('__name__'))
             if w_name is None:
                 raise oefmt(space.w_AttributeError,
                     "module has no attribute %R", w_attr)
@@ -136,7 +136,7 @@
                     "module %R has no attribute %R", w_name, w_attr)
 
     def descr_module__dir__(self, space):
-        w_dict = space.getattr(self, space.wrap('__dict__'))
+        w_dict = space.getattr(self, space.newtext('__dict__'))
         if not space.isinstance_w(w_dict, space.w_dict):
             raise oefmt(space.w_TypeError, "%N.__dict__ is not a dictionary",
                         self)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -751,8 +751,8 @@
 
         w_globals, w_locals = ensure_ns(space, w_globals, w_locals, 'exec',
                                         self)
-        space.call_method(w_globals, 'setdefault', space.wrap('__builtins__'),
-                          space.wrap(self.get_builtin()))
+        space.call_method(w_globals, 'setdefault', space.newtext('__builtins__'),
+                          self.get_builtin())
 
         plain = (self.get_w_locals() is not None and
                  space.is_w(w_locals, self.get_w_locals()))
@@ -1756,9 +1756,8 @@
 
     # internal pickling interface, not using the standard protocol
     def _get_state_(self, space):
-        w = space.wrap
-        return space.newtuple([w(self._opname), w(self.handlerposition),
-                               w(self.valuestackdepth)])
+        return space.newtuple([space.newtext(self._opname), space.newint(self.handlerposition),
+                               space.newint(self.valuestackdepth)])
 
     def handle(self, frame, unroller):
         """ Purely abstract method
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -93,7 +93,7 @@
         else:
             substr = decode_unicode_utf8(space, s, ps, q)
         v = unicodehelper.decode_unicode_escape(space, substr)
-        return space.wrap(v)
+        return space.newunicode(v)
 
     assert 0 <= ps <= q
     substr = s[ps : q]
@@ -112,7 +112,7 @@
             return W_FString(substr, rawmode)
         else:
             v = unicodehelper.decode_utf8(space, substr)
-            return space.wrap(v)
+            return space.newunicode(v)
 
     v = PyString_DecodeEscape(space, substr, 'strict', encoding)
     return space.newbytes(v)
@@ -261,9 +261,9 @@
 
 def decode_utf8_recode(space, s, ps, end, recode_encoding):
     u, ps = decode_utf8(space, s, ps, end)
-    w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding)
+    w_v = unicodehelper.encode(space, space.newunicode(u), recode_encoding)
     v = space.bytes_w(w_v)
     return v, ps
 
 def raise_app_valueerror(space, msg):
-    raise OperationError(space.w_ValueError, space.wrap(msg))
+    raise OperationError(space.w_ValueError, space.newtext(msg))
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -8,8 +8,8 @@
     if encoding == 'utf-8':
         return bytes
     w_text = space.call_method(space.newbytes(bytes), "decode",
-                               space.wrap(encoding))
-    w_recoded = space.call_method(w_text, "encode", space.wrap("utf-8"))
+                               space.newtext(encoding))
+    w_recoded = space.call_method(w_text, "encode", space.newtext("utf-8"))
     return space.bytes_w(w_recoded)
 
 def _normalize_encoding(encoding):
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -120,6 +120,7 @@
         w_EnvironmentError = [EnvironmentError]
         def wrap(self, obj):
             return [obj]
+        newint = newtext = wrap
         def call_function(self, exc, w_errno, w_msg, w_filename=None):
             return (exc, w_errno, w_msg, w_filename)
     space = FakeSpace()
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -94,7 +94,7 @@
 def default_identity_hash(space, w_obj):
     w_unique_id = w_obj.immutable_unique_id(space)
     if w_unique_id is None:     # common case
-        return space.wrap(compute_identity_hash(w_obj))
+        return space.newint(compute_identity_hash(w_obj))
     else:
         return space.hash(w_unique_id)
 
@@ -296,7 +296,7 @@
         if (space.is_w(w_obj, space.w_None)
             and not space.is_w(w_cls, space.type(space.w_None))):
             #print self, w_obj, w_cls
-            return space.wrap(self)
+            return self
         else:
             try:
                 return self.fget(self, space, w_obj)
@@ -304,7 +304,7 @@
                 return w_obj.descr_call_mismatch(
                     space, '__getattribute__',
                     self.reqcls, Arguments(space, [w_obj,
-                                                   space.wrap(self.name)]))
+                                                   space.newtext(self.name)]))
 
     def descr_property_set(self, space, w_obj, w_value):
         """property.__set__(obj, value)
@@ -318,7 +318,7 @@
             w_obj.descr_call_mismatch(
                 space, '__setattr__',
                 self.reqcls, Arguments(space, [w_obj,
-                                               space.wrap(self.name),
+                                               space.newtext(self.name),
                                                w_value]))
 
     def descr_property_del(self, space, w_obj):
@@ -333,7 +333,7 @@
             w_obj.descr_call_mismatch(
                 space, '__delattr__',
                 self.reqcls, Arguments(space, [w_obj,
-                                               space.wrap(self.name)]))
+                                               space.newtext(self.name)]))
 
     def descr_get_qualname(self, space):
         if self.qualname is None:
@@ -362,6 +362,7 @@
 
 def interp_attrproperty(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
+    # YYY needs some refactoring to get rid of the wrap
     def fget(space, obj):
         return space.wrap(getattr(obj, name))
     return GetSetProperty(fget, cls=cls, doc=doc)
@@ -421,13 +422,13 @@
         """member.__get__(obj[, type]) -> value
         Read the slot 'member' of the given 'obj'."""
         if space.is_w(w_obj, space.w_None):
-            return space.wrap(self)
+            return self
         else:
             self.typecheck(space, w_obj)
             w_result = w_obj.getslotvalue(self.index)
             if w_result is None:
                 raise OperationError(space.w_AttributeError,
-                                     space.wrap(self.name)) # XXX better message
+                                     space.newtext(self.name)) # XXX better message
             return w_result
 
     def descr_member_set(self, space, w_obj, w_value):
@@ -443,7 +444,7 @@
         success = w_obj.delslotvalue(self.index)
         if not success:
             raise OperationError(space.w_AttributeError,
-                                 space.wrap(self.name)) # XXX better message
+                                 space.newtext(self.name)) # XXX better message
 
 Member.typedef = TypeDef(
     "member_descriptor",
@@ -474,7 +475,7 @@
     def descr_new(space, w_subtype, __args__):
         self = space.allocate_instance(W_Type, w_subtype)
         W_Type.__init__(self, space)
-        return space.wrap(self)
+        return self
     descr_new = func_with_new_name(descr_new, 'descr_new_%s' % W_Type.__name__)
     return interp2app(descr_new)
 
@@ -530,10 +531,10 @@
 
 # co_xxx interface emulation for built-in code objects
 def fget_co_varnames(space, code): # unwrapping through unwrap_spec
-    return space.newtuple([space.wrap(name) for name in code.getvarnames()])
+    return space.newtuple([space.newtext(name) for name in code.getvarnames()])
 
 def fget_co_argcount(space, code): # unwrapping through unwrap_spec
-    return space.wrap(code.signature().num_argnames())
+    return space.newint(code.signature().num_argnames())
 
 def fget_zero(space, code):
     return space.wrap(0)
@@ -545,7 +546,7 @@
         flags |= CO_VARARGS
     if sig.has_kwarg():
         flags |= CO_VARKEYWORDS
-    return space.wrap(flags)
+    return space.newint(flags)
 
 def fget_co_consts(space, code): # unwrapping through unwrap_spec
     w_docstring = code.getdocstring(space)
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -17,11 +17,11 @@
     def raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeDecodeError,
-                             space.newtuple([space.wrap(encoding),
+                             space.newtuple([space.newtext(encoding),
                                              space.newbytes(s),
-                                             space.wrap(startingpos),
-                                             space.wrap(endingpos),
-                                             space.wrap(msg)]))
+                                             space.newint(startingpos),
+                                             space.newint(endingpos),
+                                             space.newtext(msg)]))
     return raise_unicode_exception_decode
 
 @specialize.memo()
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -194,7 +194,10 @@
             space = self.space
             if self.w_valuedict is None:
                 raise oefmt(space.w_TypeError, "format requires a mapping")
-            w_key = space.wrap(key)
+            if do_unicode:
+                w_key = space.newunicode(key)
+            else:
+                w_key = space.newbytes(key)
             return space.getitem(self.w_valuedict, w_key)
 
         def parse_fmt(self):
@@ -335,7 +338,7 @@
             length = len(r)
             if do_unicode and isinstance(r, str):
                 # convert string to unicode using the default encoding
-                r = self.space.unicode_w(self.space.wrap(r))
+                r = self.space.unicode_w(self.space.newbytes(r))
             prec = self.prec
             if prec == -1 and self.width == 0:
                 # fast path
@@ -454,7 +457,7 @@
             if do_unicode:
                 value = self.space.unicode_w(w_value)
             else:
-                value = self.space.str_w(w_value)
+                value = self.space.bytes_w(w_value)
             self.std_wp(value)
 
         def fmt_c(self, w_value):
@@ -569,11 +572,11 @@
                 return space.newbytes(result)
             elif fmt_type == FORMAT_BYTEARRAY:
                 return space.newbytearray([c for c in result])
-            return space.wrap(result)
+            return space.newbytes(result)
     fmt = space.unicode_w(w_fmt)
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
-    return space.wrap(result)
+    return space.newunicode(result)
 
 def mod_format(space, w_format, w_values, fmt_type=FORMAT_STR):
     if space.isinstance_w(w_values, space.w_tuple):
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -232,7 +232,7 @@
 @unmarshaller(TYPE_FLOAT)
 def unmarshal_float(space, u, tc):
     return space.call_function(space.builtin.get('float'),
-                               space.wrap(u.get_pascal()))
+                               space.newtext(u.get_pascal()))
 
 @unmarshaller(TYPE_BINARY_FLOAT)
 def unmarshal_float_bin(space, u, tc):
@@ -246,9 +246,8 @@
         m.put(pack_float(w_complex.realval))
         m.put(pack_float(w_complex.imagval))
     else:
-        # XXX a bit too wrap-happy
-        w_real = space.wrap(w_complex.realval)
-        w_imag = space.wrap(w_complex.imagval)
+        w_real = space.newfloat(w_complex.realval)
+        w_imag = space.newfloat(w_complex.imagval)
         m.start(TYPE_COMPLEX)
         m.put_pascal(space.str_w(space.repr(w_real)))
         m.put_pascal(space.str_w(space.repr(w_imag)))
@@ -256,9 +255,9 @@
 @unmarshaller(TYPE_COMPLEX)
 def unmarshal_complex(space, u, tc):
     w_real = space.call_function(space.builtin.get('float'),
-                                 space.wrap(u.get_pascal()))
+                                 space.newtext(u.get_pascal()))
     w_imag = space.call_function(space.builtin.get('float'),
-                                 space.wrap(u.get_pascal()))
+                                 space.newtext(u.get_pascal()))
     w_t = space.builtin.get('complex')
     return space.call_function(w_t, w_real, w_imag)
 
@@ -371,9 +370,9 @@
     m.atom_str(TYPE_STRING, x.co_code)
     _marshal_tuple(space, x.co_consts_w, m)
     _marshal_tuple(space, x.co_names_w, m)   # list of w_unicodes
-    co_varnames_w = [space.wrap(_decode_utf8(space, s)) for s in x.co_varnames]
-    co_freevars_w = [space.wrap(_decode_utf8(space, s)) for s in x.co_freevars]
-    co_cellvars_w = [space.wrap(_decode_utf8(space, s)) for s in x.co_cellvars]
+    co_varnames_w = [space.newunicode(_decode_utf8(space, s)) for s in x.co_varnames]
+    co_freevars_w = [space.newunicode(_decode_utf8(space, s)) for s in x.co_freevars]
+    co_cellvars_w = [space.newunicode(_decode_utf8(space, s)) for s in x.co_cellvars]
     _marshal_tuple(space, co_varnames_w, m)  # more lists, now of w_unicodes
     _marshal_tuple(space, co_freevars_w, m)
     _marshal_tuple(space, co_cellvars_w, m)
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -48,6 +48,13 @@
     class TemplateFormatter(object):
         is_unicode = for_unicode
 
+        if for_unicode:
+            def wrap(self, u):
+                return self.space.newunicode(u)
+        else:
+            def wrap(self, s):
+                return self.space.newbytes(s)
+
         parser_list_w = None
 
         def __init__(self, space, template):
@@ -104,7 +111,7 @@
                             assert end_literal > last_literal
                             literal = self.template[last_literal:end_literal]
                             w_entry = space.newtuple([
-                                space.wrap(literal),
+                                self.wrap(literal),
                                 space.w_None, space.w_None, space.w_None])
                             self.parser_list_w.append(w_entry)
                             self.last_end = i
@@ -249,7 +256,7 @@
                     if start == i:
                         raise oefmt(space.w_ValueError,
                                     "Empty attribute in format string")
-                    w_attr = space.wrap(name[start:i])
+                    w_attr = self.wrap(name[start:i])
                     if w_obj is not None:
                         w_obj = space.getattr(w_obj, w_attr)
                     else:
@@ -269,9 +276,9 @@
                         raise oefmt(space.w_ValueError, "Missing ']'")
                     index, reached = _parse_int(self.space, name, start, i)
                     if index != -1 and reached == i:
-                        w_item = space.wrap(index)
+                        w_item = space.newint(index)
                     else:
-                        w_item = space.wrap(name[start:i])
+                        w_item = self.wrap(name[start:i])
                     i += 1 # Skip "]"
                     if w_obj is not None:
                         w_obj = space.getitem(w_obj, w_item)
@@ -300,9 +307,9 @@
                 if stop != i:
                     index = -1
             if index >= 0:
-                w_first = space.wrap(index)
+                w_first = space.newint(index)
             else:
-                w_first = space.wrap(name[:i])
+                w_first = self.wrap(name[:i])
             #
             self.parser_list_w = []
             self._resolve_lookups(None, name, i, end)
@@ -335,11 +342,15 @@
                     space = self.space
                     startm1 = start - 1
                     assert startm1 >= self.last_end
+                    if conversion is None:
+                        w_conversion = space.w_None
+                    else:
+                        w_conversion = self.wrap(conversion)
                     w_entry = space.newtuple([
-                        space.wrap(self.template[self.last_end:startm1]),
-                        space.wrap(name),
-                        space.wrap(spec),
-                        space.wrap(conversion)])
+                        self.wrap(self.template[self.last_end:startm1]),
+                        self.wrap(name),
+                        self.wrap(spec),
+                        w_conversion])
                     self.parser_list_w.append(w_entry)
                     self.last_end = end + 1
                 return self.empty
@@ -349,7 +360,7 @@
                 w_obj = self._convert(w_obj, conversion)
             if recursive:
                 spec = self._build_string(spec_start, end, level)
-            w_rendered = self.space.format(w_obj, self.space.wrap(spec))
+            w_rendered = self.space.format(w_obj, self.wrap(spec))
             unwrapper = "unicode_w" if self.is_unicode else "bytes_w"
             to_interp = getattr(self.space, unwrapper)
             return to_interp(w_rendered)
@@ -362,7 +373,7 @@
             space = self.space
             if self.last_end < len(self.template):
                 w_lastentry = space.newtuple([
-                    space.wrap(self.template[self.last_end:]),
+                    self.wrap(self.template[self.last_end:]),
                     space.w_None,
                     space.w_None,
                     space.w_None])
@@ -378,10 +389,10 @@
     if is_unicode:
         template = unicode_template_formatter(space,
                                               space.unicode_w(w_string))
-        return space.wrap(template.build(args, w_kwargs))
+        return space.newunicode(template.build(args, w_kwargs))
     else:
         template = str_template_formatter(space, space.bytes_w(w_string))
-        return space.wrap(template.build(args, w_kwargs))
+        return space.newbytes(template.build(args, w_kwargs))
 
 
 class NumberSpec(object):
@@ -412,6 +423,13 @@
     class Formatter(BaseFormatter):
         """__format__ implementation for builtin types."""
 
+        if for_unicode:
+            def wrap(self, u):
+                return self.space.newunicode(u)
+        else:
+            def wrap(self, s):
+                return self.space.newbytes(s)
+
         is_unicode = for_unicode
         _grouped_digits = None
 
@@ -557,7 +575,7 @@
                 w_string = space.str(w_string)
             string = space.unicode_w(w_string)
             if self._parse_spec("s", "<"):
-                return space.wrap(string)
+                return self.wrap(string)
             if self._type != "s":
                 self._unknown_presentation("string")
             if self._sign != "\0":
@@ -578,7 +596,7 @@
                 length = precision
                 string = string[:precision]
             self._calc_padding(string, length)
-            return space.wrap(self._pad(string))
+            return self.wrap(self._pad(string))
 
         def _get_locale(self, tp):
             if tp == "n":
@@ -830,8 +848,8 @@
                                         n_remainder, False, result)
             fill = self._fill_char
             upper = self._type == "X"
-            return self.space.wrap(self._fill_number(spec, result, to_numeric,
-                                     to_prefix, fill, to_remainder, upper))
+            return self.wrap(self._fill_number(spec, result, to_numeric,
+                             to_prefix, fill, to_remainder, upper))
 
         def _long_to_base(self, base, value):
             prefix = ""
@@ -981,8 +999,8 @@
             spec = self._calc_num_width(0, sign, to_number, n_digits,
                                         n_remainder, have_dec_point, digits)
             fill = self._fill_char
-            return self.space.wrap(self._fill_number(spec, digits, to_number, 0,
-                                      fill, to_remainder, False))
+            return self.wrap(self._fill_number(spec, digits, to_number, 0,
+                             fill, to_remainder, False))
 
         def format_float(self, w_float):
             space = self.space
@@ -1141,7 +1159,7 @@
             #add right padding
             out.append_multiple_char(fill, self._right_pad)
 
-            return self.space.wrap(out.build())
+            return self.wrap(out.build())
 
 
         def format_complex(self, w_complex):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -166,18 +166,7 @@
             #print 'wrapping', x, '->', w_result
             return w_result
         if isinstance(x, base_int):
-            if self.config.objspace.std.withsmalllong:
-                from pypy.objspace.std.smalllongobject import W_SmallLongObject
-                from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
-                from rpython.rlib.rarithmetic import longlongmax
-                if (not isinstance(x, r_ulonglong)
-                    or x <= r_ulonglong(longlongmax)):
-                    return W_SmallLongObject(r_longlong(x))
-            x = widen(x)
-            if isinstance(x, int):
-                return self.newint(x)
-            else:
-                return W_LongObject.fromrarith_int(x)
+            return self.newint(x)
         return self._wrap_not_rpython(x)
 
     def _wrap_ascii_replace(self, x):
@@ -275,7 +264,18 @@
             return w_obj.unwrap(self)
         raise TypeError("cannot unwrap: %r" % w_obj)
 
+    @specialize.argtype(1)
     def newint(self, intval):
+        if self.config.objspace.std.withsmalllong:
+            from pypy.objspace.std.smalllongobject import W_SmallLongObject
+            from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
+            from rpython.rlib.rarithmetic import longlongmax
+            if (not isinstance(intval, r_ulonglong)
+                or intval <= r_ulonglong(longlongmax)):
+                return W_SmallLongObject(r_longlong(intval))
+        intval = widen(intval)
+        if not isinstance(intval, int):
+            return W_LongObject.fromrarith_int(intval)
         return wrapint(self, intval)
 
     def newfloat(self, floatval):
@@ -370,6 +370,11 @@
     def newtext(self, s):
         return self.newunicode(decode_utf8(self, s, allow_surrogates=True))
 
+    def newtext_or_none(self, s):
+        if s is None:
+            return self.w_None
+        return self.newtext(s)
+
     def newunicode(self, uni):
         assert isinstance(uni, unicode)
         return W_UnicodeObject(uni)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1154,7 +1154,7 @@
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
-        w_self.dict_w[slot_name] = space.wrap(member)
+        w_self.dict_w[slot_name] = member
         return True
     else:
         w_prev = w_self.dict_w[slot_name]
@@ -1168,15 +1168,13 @@
 def create_dict_slot(w_self):
     if not w_self.hasdict:
         descr = dict_descr.copy_for_type(w_self)
-        w_self.dict_w.setdefault('__dict__',
-                                 w_self.space.wrap(descr))
+        w_self.dict_w.setdefault('__dict__', descr)
         w_self.hasdict = True
 
 def create_weakref_slot(w_self):
     if not w_self.weakrefable:
         descr = weakref_descr.copy_for_type(w_self)
-        w_self.dict_w.setdefault('__weakref__',
-                                 w_self.space.wrap(descr))
+        w_self.dict_w.setdefault('__weakref__', descr)
         w_self.weakrefable = True
 
 def setup_user_defined_type(w_self, force_new_layout):
@@ -1201,7 +1199,11 @@
 def setup_builtin_type(w_self, instancetypedef):
     w_self.hasdict = instancetypedef.hasdict
     w_self.weakrefable = instancetypedef.weakrefable
-    w_self.w_doc = w_self.space.wrap(instancetypedef.doc)
+    if isinstance(instancetypedef.doc, W_Root):
+        w_doc = instancetypedef.doc
+    else:
+        w_doc = w_self.space.newtext_or_none(instancetypedef.doc)
+    w_self.w_doc = w_doc
     ensure_common_attributes(w_self)
     #
     # usually 'instancetypedef' is new, i.e. not seen in any base,
@@ -1393,7 +1395,7 @@
                               overridetypedef=overridetypedef,
                               is_heaptype=overridetypedef.heaptype)
         if typedef is not overridetypedef:
-            w_type.w_doc = space.wrap(typedef.doc)
+            w_type.w_doc = space.newtext_or_none(typedef.doc)
         else:
             # Set the __qualname__ of member functions
             for name in rawdict:
@@ -1401,8 +1403,9 @@
                 if isinstance(w_obj, ClassMethod):
                     w_obj = w_obj.w_function
                 if isinstance(w_obj, FunctionWithFixedCode):
-                    qualname = w_type.getqualname(space) + '.' + name
-                    w_obj.fset_func_qualname(space, space.wrap(qualname))
+                    qualname = (w_type.getqualname(space).encode('utf-8')
+                                + '.' + name)
+                    w_obj.fset_func_qualname(space, space.newtext(qualname))
 
         if hasattr(typedef, 'flag_sequence_bug_compat'):
             w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat

From pypy.commits at gmail.com  Mon Feb 13 12:44:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 09:44:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix
Message-ID: <58a1f071.4a371c0a.a6132.fb42@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90097:b6513a7b1305
Date: 2017-02-13 18:43 +0100
http://bitbucket.org/pypy/pypy/changeset/b6513a7b1305/

Log:	fix

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -31,7 +31,7 @@
         return _wrapkey(self.space, key)
 
     def unwrap(self, wrapped):
-        return self.space.str_w(wrapped)
+        return self.space.text_w(wrapped)
 
     def get_empty_storage(self):
         d = ([], [])

From pypy.commits at gmail.com  Mon Feb 13 12:47:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 09:47:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix
Message-ID: <58a1f13d.c1acdf0a.8c76e.e444@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90098:144b2aec56bd
Date: 2017-02-13 18:47 +0100
http://bitbucket.org/pypy/pypy/changeset/144b2aec56bd/

Log:	fix

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -741,7 +741,7 @@
         space = self.space
         w_lookup_type = space.type(w_key)
         if space.is_w(w_lookup_type, space.w_unicode):
-            return self.getitem_str(w_dict, space.str_w(w_key))
+            return self.getitem_str(w_dict, space.text_w(w_key))
         elif _never_equal_to_string(space, w_lookup_type):
             return None
         else:
@@ -760,7 +760,7 @@
     def setitem(self, w_dict, w_key, w_value):
         space = self.space
         if space.is_w(space.type(w_key), space.w_unicode):
-            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
+            self.setitem_str(w_dict, self.space.text_w(w_key), w_value)
         else:
             self.switch_to_object_strategy(w_dict)
             w_dict.setitem(w_key, w_value)
@@ -768,7 +768,7 @@
     def setdefault(self, w_dict, w_key, w_default):
         space = self.space
         if space.is_w(space.type(w_key), space.w_unicode):
-            key = space.str_w(w_key)
+            key = space.text_w(w_key)
             w_result = self.getitem_str(w_dict, key)
             if w_result is not None:
                 return w_result
@@ -783,7 +783,7 @@
         w_key_type = space.type(w_key)
         w_obj = self.unerase(w_dict.dstorage)
         if space.is_w(w_key_type, space.w_unicode):
-            key = self.space.str_w(w_key)
+            key = self.space.text_w(w_key)
             flag = w_obj.deldictvalue(space, key)
             if not flag:
                 raise KeyError

From pypy.commits at gmail.com  Mon Feb 13 12:59:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 09:59:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 5fef2e96ef36
Message-ID: <58a1f3e7.128bdf0a.2117c.dd40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90099:135e906c0b0c
Date: 2017-02-13 18:52 +0100
http://bitbucket.org/pypy/pypy/changeset/135e906c0b0c/

Log:	hg merge 5fef2e96ef36

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -630,7 +630,7 @@
                 if not e.match(self, self.w_StopIteration):
                     raise
                 break
-            name = self.str_w(w_name)
+            name = self.text_w(w_name)
             if not name.startswith('__'):
                 excname = name
                 w_exc = self.getitem(w_dic, w_name)
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -26,7 +26,7 @@
 # helper
 
 def unpack_str_tuple(space,w_str_tuple):
-    return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
+    return [space.text_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
 
 # Magic numbers for the bytecode version in code objects.
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -95,10 +95,13 @@
     def wrap(self, obj):
         return obj
     newtext = wrap
+    newunicode = wrap
 
     def str_w(self, s):
         return str(s)
-    text_w = str_w
+
+    def text_w(self, s):
+        return self.str_w(s)
 
     def unicode_w(self, s):
         return unicode(s)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -26,8 +26,8 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        s1 = space.str_w(self)
-        s2 = space.str_w(w_other)
+        s1 = space.bytes_w(self)
+        s2 = space.bytes_w(w_other)
         if len(s2) > 1:
             return s1 is s2
         else:            # strings of len <= 1 are unique-ified
@@ -36,7 +36,7 @@
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        s = space.str_w(self)
+        s = space.bytes_w(self)
         if len(s) > 1:
             uid = compute_unique_id(s)
         else:            # strings of len <= 1 are unique-ified
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -518,7 +518,7 @@
         return _new_int(space, w_inttype, w_x, w_base)
 
     def descr_hash(self, space):
-        return space.wrap(_hash_int(self.intval))
+        return space.newint(_hash_int(self.intval))
 
     def as_w_long(self, space):
         return space.newlong(self.intval)

From pypy.commits at gmail.com  Mon Feb 13 12:59:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 09:59:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix
Message-ID: <58a1f3e9.56711c0a.ac977.164a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90100:cef36504d2aa
Date: 2017-02-13 18:58 +0100
http://bitbucket.org/pypy/pypy/changeset/cef36504d2aa/

Log:	fix

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -39,6 +39,7 @@
 
 class RUnicodeEncodeError(Exception):
     def __init__(self, encoding, object, start, end, reason):
+        assert isinstance(object, unicode)
         self.encoding = encoding
         self.object = object
         self.start = start
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -601,7 +601,7 @@
     raise OperationError(space.w_UnicodeEncodeError,
                          space.newtuple([
         space.newtext(ue.encoding),
-        space.newbytes(ue.object),
+        space.newunicode(ue.object),
         space.newint(ue.start),
         space.newint(ue.end),
         space.newtext(ue.reason)]))

From pypy.commits at gmail.com  Mon Feb 13 13:14:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 13 Feb 2017 10:14:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 1bddaf521a03
Message-ID: <58a1f79f.c69bdf0a.df5f5.ee71@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90101:f73ab0b44844
Date: 2017-02-13 19:13 +0100
http://bitbucket.org/pypy/pypy/changeset/f73ab0b44844/

Log:	hg merge 1bddaf521a03

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -87,7 +87,7 @@
         # this is obscure and slow
         space = self.space
         try:
-            w_builtin = space.getitem(w_globals, space.wrap('__builtins__'))
+            w_builtin = space.getitem(w_globals, space.newtext('__builtins__'))
         except OperationError as e:
             if not e.match(space, space.w_KeyError):
                 raise
@@ -100,7 +100,7 @@
                 return w_builtin
         # no builtin! make a default one.  Give them None, at least.
         builtin = module.Module(space, None)
-        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
+        space.setitem(builtin.w_dict, space.newtext('None'), space.w_None)
         return builtin
 
     def setup_after_space_initialization(self):
diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -17,7 +17,7 @@
     no __bases__ or if cls.__bases__ is not a tuple.
     """
     try:
-        w_bases = space.getattr(w_cls, space.wrap('__bases__'))
+        w_bases = space.getattr(w_cls, space.newtext('__bases__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise       # propagate other errors
@@ -32,12 +32,12 @@
 
 def check_class(space, w_obj, msg):
     if not abstract_isclass_w(space, w_obj):
-        raise OperationError(space.w_TypeError, space.wrap(msg))
+        raise OperationError(space.w_TypeError, space.newtext(msg))
 
 
 def abstract_getclass(space, w_obj):
     try:
-        return space.getattr(w_obj, space.wrap('__class__'))
+        return space.getattr(w_obj, space.newtext('__class__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise       # propagate other errors
@@ -55,7 +55,7 @@
     check_class(space, w_cls, "isinstance() arg 2 must be a class, type,"
                               " or tuple of classes and types")
     try:
-        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+        w_abstractclass = space.getattr(w_inst, space.newtext('__class__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise       # propagate other errors
@@ -70,7 +70,7 @@
     if space.isinstance_w(w_inst, w_type):
         return True
     try:
-        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+        w_abstractclass = space.getattr(w_inst, space.newtext('__class__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise       # propagate other errors
@@ -212,14 +212,14 @@
 another class.  When using a tuple as the second argument, check whether
 'cls' is a subclass of any of the classes listed in the tuple."""
     result = abstract_issubclass_w(space, w_cls, w_klass_or_tuple, True)
-    return space.wrap(result)
+    return space.newbool(result)
 
 def isinstance(space, w_obj, w_klass_or_tuple):
     """Check whether an object is an instance of a class (or of a subclass
 thereof).  When using a tuple as the second argument, check whether 'obj'
 is an instance of any of the classes listed in the tuple."""
     result = abstract_isinstance_w(space, w_obj, w_klass_or_tuple, True)
-    return space.wrap(result)
+    return space.newbool(result)
 
 # avoid namespace pollution
 app_issubclass = issubclass; del issubclass
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -45,9 +45,8 @@
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
         ast_node = ast.mod.from_object(space, w_source)
         ec.compiler.validate_ast(ast_node)
-        code = ec.compiler.compile_ast(ast_node, filename, mode, flags,
+        return ec.compiler.compile_ast(ast_node, filename, mode, flags,
                                        optimize=optimize)
-        return space.wrap(code)
 
     flags |= consts.PyCF_SOURCE_IS_UTF8
     source, flags = source_as_str(space, w_source, 'compile',
@@ -57,9 +56,8 @@
         node = ec.compiler.compile_to_ast(source, filename, mode, flags)
         return node.to_object(space)
     else:
-        code = ec.compiler.compile(source, filename, mode, flags,
+        return ec.compiler.compile(source, filename, mode, flags,
                                    optimize=optimize)
-        return space.wrap(code)
 
 
 def eval(space, w_prog, w_globals=None, w_locals=None):
@@ -113,7 +111,7 @@
         w_meta = _calculate_metaclass(space, w_meta, bases_w)
 
     try:
-        w_prep = space.getattr(w_meta, space.wrap("__prepare__"))
+        w_prep = space.getattr(w_meta, space.newtext("__prepare__"))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -33,7 +33,7 @@
             objtype_name = u"<%s object>" % self.w_objtype.getname(space)
         else:
             objtype_name = u'NULL'
-        return space.wrap(u"<super: <class '%s'>, %s>" % (
+        return space.newunicode(u"<super: <class '%s'>, %s>" % (
             self.w_starttype.getname(space), objtype_name))
 
     def get(self, space, w_obj, w_type=None):
@@ -122,7 +122,7 @@
         return w_objtype
 
     try:
-        w_type = space.getattr(w_obj_or_type, space.wrap('__class__'))
+        w_type = space.getattr(w_obj_or_type, space.newtext('__class__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -175,12 +175,12 @@
         # our __doc__ comes from the getter if we don't have an explicit one
         if (space.is_w(self.w_doc, space.w_None) and
             not space.is_w(self.w_fget, space.w_None)):
-            w_getter_doc = space.findattr(self.w_fget, space.wrap('__doc__'))
+            w_getter_doc = space.findattr(self.w_fget, space.newtext('__doc__'))
             if w_getter_doc is not None:
                 if type(self) is W_Property:
                     self.w_doc = w_getter_doc
                 else:
-                    space.setattr(self, space.wrap('__doc__'), w_getter_doc)
+                    space.setattr(self, space.newtext('__doc__'), w_getter_doc)
                 self.getter_doc = True
 
     def get(self, space, w_obj, w_objtype=None):
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -251,10 +251,10 @@
 
         self = space.allocate_instance(W_Enumerate, w_subtype)
         self.__init__(w_iter, start, w_start)
-        return space.wrap(self)
+        return self
 
     def descr___iter__(self, space):
-        return space.wrap(self)
+        return self
 
     def descr_next(self, space):
         from pypy.objspace.std.listobject import W_ListObject
@@ -276,14 +276,14 @@
                 try:
                     newval = rarithmetic.ovfcheck(index + 1)
                 except OverflowError:
-                    w_index = space.wrap(index)
-                    self.w_index = space.add(w_index, space.wrap(1))
+                    w_index = space.newint(index)
+                    self.w_index = space.add(w_index, space.newint(1))
                     self.index = -1
                 else:
                     self.index = newval
-            w_index = space.wrap(index)
+            w_index = space.newint(index)
         else:
-            self.w_index = space.add(w_index, space.wrap(1))
+            self.w_index = space.add(w_index, space.newint(1))
         if w_item is None:
             w_item = space.next(self.w_iter_or_list)
         return space.newtuple([w_index, w_item])
@@ -291,7 +291,7 @@
     def descr___reduce__(self, space):
         w_index = self.w_index
         if w_index is None:
-            w_index = space.wrap(self.index)
+            w_index = space.newint(self.index)
         return space.newtuple([space.type(self),
                                space.newtuple([self.w_iter_or_list, w_index])])
 
@@ -302,7 +302,7 @@
         w_index = None
     else:
         index = -1
-    return space.wrap(W_Enumerate(w_iter_or_list, index, w_index))
+    return W_Enumerate(w_iter_or_list, index, w_index)
 
 W_Enumerate.typedef = TypeDef("enumerate",
     __new__=interp2app(W_Enumerate.descr___new__),
@@ -333,10 +333,10 @@
                         "argument to reversed() must be a sequence")
         self = space.allocate_instance(W_ReversedIterator, w_subtype)
         self.__init__(space, w_sequence)
-        return space.wrap(self)
+        return self
 
     def descr___iter__(self, space):
-        return space.wrap(self)
+        return self
 
     def descr_length_hint(self, space):
         # bah, there is even a CPython test that checks that this
@@ -347,11 +347,11 @@
             rem_length = self.remaining + 1
             if rem_length <= total_length:
                 res = rem_length
-        return space.wrap(res)
+        return space.newint(res)
 
     def descr_next(self, space):
         if self.remaining >= 0:
-            w_index = space.wrap(self.remaining)
+            w_index = space.newint(self.remaining)
             try:
                 w_item = space.getitem(self.w_sequence, w_index)
             except OperationError as e:
@@ -373,7 +373,7 @@
 
     def descr___reduce__(self, space):
         if self.w_sequence:
-            w_state = space.wrap(self.remaining)
+            w_state = space.newint(self.remaining)
             return space.newtuple([
                 space.type(self),
                 space.newtuple([self.w_sequence]),
@@ -413,7 +413,7 @@
         w_start = space.index(w_start)
         promote_step = False
         if space.is_none(w_step):  # no step argument provided
-            w_step = space.wrap(1)
+            w_step = space.newint(1)
             promote_step = True
         if space.is_none(w_stop):  # only 1 argument provided
             w_start, w_stop = space.newint(0), w_start
@@ -431,15 +431,15 @@
         w_length = compute_range_length(space, w_start, w_stop, w_step)
         obj = space.allocate_instance(W_Range, w_subtype)
         W_Range.__init__(obj, w_start, w_stop, w_step, w_length, promote_step)
-        return space.wrap(obj)
+        return obj
 
     def descr_repr(self, space):
         if not space.is_true(space.eq(self.w_step, space.newint(1))):
-            return space.mod(space.wrap("range(%d, %d, %d)"),
+            return space.mod(space.newtext("range(%d, %d, %d)"),
                              space.newtuple([self.w_start, self.w_stop,
                                              self.w_step]))
         else:
-            return space.mod(space.wrap("range(%d, %d)"),
+            return space.mod(space.newtext("range(%d, %d)"),
                              space.newtuple([self.w_start, self.w_stop]))
 
     def descr_len(self):
@@ -473,7 +473,7 @@
 
         w_length = compute_range_length(space, w_substart, w_substop, w_substep)
         obj = W_Range(w_substart, w_substop, w_substep, w_length)
-        return space.wrap(obj)
+        return obj
 
     def descr_getitem(self, space, w_index):
         # Cannot use the usual space.decode_index methods, because
@@ -504,8 +504,8 @@
             self.w_start,
             space.mul(space.sub(self.w_length, space.newint(1)),
                       self.w_step))
-        return space.wrap(W_LongRangeIterator(
-                space, w_lastitem, space.neg(self.w_step), self.w_length))
+        return W_LongRangeIterator(
+                space, w_lastitem, space.neg(self.w_step), self.w_length)
 
     def descr_reduce(self, space):
         return space.newtuple(
@@ -564,18 +564,18 @@
             return space.w_NotImplemented
         if not space.eq_w(self.w_length, w_other.w_length):
             return space.w_False
-        if space.eq_w(self.w_length, space.wrap(0)):
+        if space.eq_w(self.w_length, space.newint(0)):
             return space.w_True
         if not space.eq_w(self.w_start, w_other.w_start):
             return space.w_False
-        if space.eq_w(self.w_length, space.wrap(1)):
+        if space.eq_w(self.w_length, space.newint(1)):
             return space.w_True
         return space.eq(self.w_step, w_other.w_step)
 
     def descr_hash(self, space):
-        if space.eq_w(self.w_length, space.wrap(0)):
+        if space.eq_w(self.w_length, space.newint(0)):
             w_tup = space.newtuple([self.w_length, space.w_None, space.w_None])
-        elif space.eq_w(self.w_length, space.wrap(1)):
+        elif space.eq_w(self.w_length, space.newint(1)):
             w_tup = space.newtuple([self.w_length, self.w_start, space.w_None])
         else:
             w_tup = space.newtuple([self.w_length, self.w_start, self.w_step])
@@ -605,7 +605,7 @@
 class W_AbstractRangeIterator(W_Root):
 
     def descr_iter(self, space):
-        return space.wrap(self)
+        return self
 
     def descr_len(self, space):
         raise NotImplementedError
@@ -662,7 +662,7 @@
             item = self.current
             self.current = item + self.step
             self.remaining -= 1
-            return space.wrap(item)
+            return space.newint(item)
         raise OperationError(space.w_StopIteration, space.w_None)
 
     def descr_len(self, space):
@@ -673,14 +673,13 @@
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
         new_inst = mod.get('intrangeiter_new')
-        w        = space.wrap
         nt = space.newtuple
 
-        tup = [w(self.current), self.get_remaining(space), w(self.step)]
+        tup = [space.newint(self.current), self.get_remaining(space), space.newint(self.step)]
         return nt([new_inst, nt(tup)])
 
     def get_remaining(self, space):
-        return space.wrap(self.remaining)
+        return space.newint(self.remaining)
 
 
 class W_IntRangeStepOneIterator(W_IntRangeIterator):
@@ -695,11 +694,11 @@
         if self.current < self.stop:
             item = self.current
             self.current = item + 1
-            return space.wrap(item)
+            return space.newint(item)
         raise OperationError(space.w_StopIteration, space.w_None)
 
     def get_remaining(self, space):
-        return space.wrap(self.stop - self.current)
+        return space.newint(self.stop - self.current)
 
 
 W_AbstractRangeIterator.typedef = TypeDef("rangeiterator",
@@ -739,7 +738,7 @@
         self.iterators_w = iterators_w
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         # common case: 1 or 2 arguments
@@ -764,7 +763,7 @@
 
     def descr_reduce(self, space):
         w_map = space.getattr(space.getbuiltinmodule('builtins'),
-                space.wrap('map'))
+                space.newtext('map'))
         args_w = [self.w_fun] + self.iterators_w
         return space.newtuple([w_map, space.newtuple(args_w)])
 
@@ -775,7 +774,7 @@
                     "map() must have at least two arguments")
     r = space.allocate_instance(W_Map, w_subtype)
     r.__init__(space, w_fun, args_w)
-    return space.wrap(r)
+    return r
 
 W_Map.typedef = TypeDef(
         'map',
@@ -800,7 +799,7 @@
         self.iterable = space.iter(w_iterable)
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         while True:
@@ -815,7 +814,7 @@
 
     def descr_reduce(self, space):
         w_filter = space.getattr(space.getbuiltinmodule('builtins'),
-                space.wrap('filter'))
+                space.newtext('filter'))
         args_w = [space.w_None if self.no_predicate else self.w_predicate,
                   self.iterable]
         return space.newtuple([w_filter, space.newtuple(args_w)])
@@ -824,7 +823,7 @@
 def W_Filter___new__(space, w_subtype, w_predicate, w_iterable):
     r = space.allocate_instance(W_Filter, w_subtype)
     r.__init__(space, w_predicate, w_iterable)
-    return space.wrap(r)
+    return r
 
 W_Filter.typedef = TypeDef(
         'filter',
@@ -850,14 +849,14 @@
 
     def descr_reduce(self, space):
         w_zip = space.getattr(space.getbuiltinmodule('builtins'),
-                space.wrap('zip'))
+                space.newtext('zip'))
         return space.newtuple([w_zip, space.newtuple(self.iterators_w)])
 
 
 def W_Zip___new__(space, w_subtype, args_w):
     r = space.allocate_instance(W_Zip, w_subtype)
     r.__init__(space, None, args_w)
-    return space.wrap(r)
+    return r
 
 W_Zip.typedef = TypeDef(
         'zip',
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -29,7 +29,7 @@
         c = UNICHR(code)
     except ValueError:
         raise oefmt(space.w_ValueError, "chr() arg out of range")
-    return space.wrap(c)
+    return space.newunicode(c)
 
 def len(space, w_obj):
     "len(object) -> integer\n\nReturn the number of items of a sequence or mapping."
@@ -44,8 +44,8 @@
     # Note that if w_name is already an exact string it must be returned
     # unmodified (and not e.g. unwrapped-rewrapped).
     if not space.is_w(space.type(w_name), space.w_text):
-        name = space.unicode_w(w_name)  # typecheck
-        w_name = space.wrap(name)  # rewrap as a real string
+        name = space.text_w(w_name)  # typecheck
+        w_name = space.newtext(name)  # rewrap as a real string
     return w_name
 
 def delattr(space, w_object, w_name):

From pypy.commits at gmail.com  Mon Feb 13 14:40:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Feb 2017 11:40:23 -0800 (PST)
Subject: [pypy-commit] pypy default: Define PyCFunctionObject using cparser
Message-ID: <58a20ba7.128bdf0a.2117c.fc6a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90102:0ee3e96efc24
Date: 2017-02-13 19:39 +0000
http://bitbucket.org/pypy/pypy/changeset/0ee3e96efc24/

Log:	Define PyCFunctionObject using cparser

diff --git a/pypy/module/cpyext/include/methodobject.h b/pypy/module/cpyext/include/methodobject.h
--- a/pypy/module/cpyext/include/methodobject.h
+++ b/pypy/module/cpyext/include/methodobject.h
@@ -7,14 +7,6 @@
 extern "C" {
 #endif
 
-typedef struct
-{
-    PyObject_HEAD
-    PyMethodDef *m_ml; /* Description of the C function to call */
-    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
-    PyObject    *m_module; /* The __module__ attribute, can be anything */
-} PyCFunctionObject;
-
 /* Flag passed to newmethodobject */
 #define METH_OLDARGS  0x0000
 #define METH_VARARGS  0x0001
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -9,8 +9,8 @@
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
-    METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
-    build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
+    METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
+    build_type_checkers, cpython_api, generic_cpy_call,
     PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
@@ -18,14 +18,7 @@
 PyMethodDef = cts.gettype('PyMethodDef')
 PyCFunction = cts.gettype('PyCFunction')
 PyCFunctionKwArgs = cts.gettype('PyCFunctionWithKeywords')
-PyCFunctionObjectStruct = cpython_struct(
-    'PyCFunctionObject',
-    PyObjectFields + (
-     ('m_ml', lltype.Ptr(PyMethodDef)),
-     ('m_self', PyObject),
-     ('m_module', PyObject),
-     ))
-PyCFunctionObject = lltype.Ptr(PyCFunctionObjectStruct)
+PyCFunctionObject = cts.gettype('PyCFunctionObject*')
 
 @bootstrap_function
 def init_methodobject(space):
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -213,6 +213,13 @@
 };
 typedef struct PyMethodDef PyMethodDef;
 
+typedef struct {
+    PyObject_HEAD
+    PyMethodDef *m_ml; /* Description of the C function to call */
+    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
+    PyObject    *m_module; /* The __module__ attribute, can be anything */
+} PyCFunctionObject;
+
 /* from structmember.h */
 typedef struct PyMemberDef {
     /* Current version, use this */

From pypy.commits at gmail.com  Mon Feb 13 14:41:54 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Feb 2017 11:41:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a20c02.44641c0a.c8520.397e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90103:36c29756f6b7
Date: 2017-02-13 19:41 +0000
http://bitbucket.org/pypy/pypy/changeset/36c29756f6b7/

Log:	hg merge default

diff --git a/pypy/module/cpyext/include/methodobject.h b/pypy/module/cpyext/include/methodobject.h
--- a/pypy/module/cpyext/include/methodobject.h
+++ b/pypy/module/cpyext/include/methodobject.h
@@ -7,14 +7,6 @@
 extern "C" {
 #endif
 
-typedef struct
-{
-    PyObject_HEAD
-    PyMethodDef *m_ml; /* Description of the C function to call */
-    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
-    PyObject    *m_module; /* The __module__ attribute, can be anything */
-} PyCFunctionObject;
-
 /* Flag passed to newmethodobject */
 #define METH_VARARGS  0x0001
 #define METH_KEYWORDS 0x0002
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -9,8 +9,8 @@
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
-    METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
-    build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
+    METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
+    build_type_checkers, cpython_api, generic_cpy_call,
     PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
@@ -18,14 +18,7 @@
 PyMethodDef = cts.gettype('PyMethodDef')
 PyCFunction = cts.gettype('PyCFunction')
 PyCFunctionKwArgs = cts.gettype('PyCFunctionWithKeywords')
-PyCFunctionObjectStruct = cpython_struct(
-    'PyCFunctionObject',
-    PyObjectFields + (
-     ('m_ml', lltype.Ptr(PyMethodDef)),
-     ('m_self', PyObject),
-     ('m_module', PyObject),
-     ))
-PyCFunctionObject = lltype.Ptr(PyCFunctionObjectStruct)
+PyCFunctionObject = cts.gettype('PyCFunctionObject*')
 
 @bootstrap_function
 def init_methodobject(space):
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -188,6 +188,13 @@
 };
 typedef struct PyMethodDef PyMethodDef;
 
+typedef struct {
+    PyObject_HEAD
+    PyMethodDef *m_ml; /* Description of the C function to call */
+    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
+    PyObject    *m_module; /* The __module__ attribute, can be anything */
+} PyCFunctionObject;
+
 /* from structmember.h */
 typedef struct PyMemberDef {
     /* Current version, use this */

From pypy.commits at gmail.com  Mon Feb 13 14:53:54 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Feb 2017 11:53:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use 3.5 definition for PyCFunctionObject
 (adds m_weakreflist)
Message-ID: <58a20ed2.f0a6df0a.b1e14.0f73@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90104:bdc8a83774b0
Date: 2017-02-13 19:53 +0000
http://bitbucket.org/pypy/pypy/changeset/bdc8a83774b0/

Log:	Use 3.5 definition for PyCFunctionObject (adds m_weakreflist)

diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -193,6 +193,7 @@
     PyMethodDef *m_ml; /* Description of the C function to call */
     PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
     PyObject    *m_module; /* The __module__ attribute, can be anything */
+    PyObject    *m_weakreflist; /* List of weak references */
 } PyCFunctionObject;
 
 /* from structmember.h */

From pypy.commits at gmail.com  Mon Feb 13 16:25:49 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 13 Feb 2017 13:25:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix translation (errmsg must be str,
 not unicode)
Message-ID: <58a2245d.977d1c0a.7b97f.5325@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90105:5ddabb9320f9
Date: 2017-02-13 21:25 +0000
http://bitbucket.org/pypy/pypy/changeset/5ddabb9320f9/

Log:	fix translation (errmsg must be str, not unicode)

diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -58,7 +58,7 @@
                 errorpos = rffi.cast(lltype.Signed, errorposp[0])
                 if errorpos == -1:
                     raise MemoryError
-                errmsg = _errmsg(u"pypy_wchar2char")
+                errmsg = _errmsg("pypy_wchar2char")
                 errorhandler('strict', 'filesystemencoding', errmsg, u,
                              errorpos, errorpos + 1)
             return rffi.charp2str(sbuf)
@@ -81,7 +81,7 @@
             ubuf = pypy_char2wchar(sbuf, sizep)
         try:
             if not ubuf:
-                errmsg = _errmsg(u"pypy_char2wchar")
+                errmsg = _errmsg("pypy_char2wchar")
                 errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
             size = rffi.cast(lltype.Signed, sizep[0])
             return rawwcharp2unicoden(ubuf, size)
@@ -91,7 +91,7 @@
 
 def _errmsg(what):
     # I *think* that the functions in locale_codec.c don't set errno
-    return u"%s failed" % what
+    return "%s failed" % what
 
 
 class scoped_unicode2rawwcharp:

From pypy.commits at gmail.com  Tue Feb 14 01:00:20 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Mon, 13 Feb 2017 22:00:20 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: Use HTTPS urls for external
 resources
Message-ID: <58a29cf4.b086df0a.ceffa.6c6c@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r859:93f1a38cc2a0
Date: 2017-02-13 21:59 -0800
http://bitbucket.org/pypy/pypy.org/changeset/93f1a38cc2a0/

Log:	Use HTTPS urls for external resources

diff --git a/archive.html b/archive.html
--- a/archive.html
+++ b/archive.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/contact.html b/contact.html
--- a/contact.html
+++ b/contact.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/howtohelp.html b/howtohelp.html
--- a/howtohelp.html
+++ b/howtohelp.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/people.html b/people.html
--- a/people.html
+++ b/people.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/performance.html b/performance.html
--- a/performance.html
+++ b/performance.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi
--- a/source/_layouts/site.genshi
+++ b/source/_layouts/site.genshi
@@ -65,10 +65,10 @@
 
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px"/></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px"/></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
diff --git a/sponsor.html b/sponsor.html
--- a/sponsor.html
+++ b/sponsor.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/success.html b/success.html
--- a/success.html
+++ b/success.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/tmdonate.html b/tmdonate.html
--- a/tmdonate.html
+++ b/tmdonate.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
diff --git a/tmdonate2.html b/tmdonate2.html
--- a/tmdonate2.html
+++ b/tmdonate2.html
@@ -31,10 +31,10 @@
 <div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
 <div id="header">
     <div id="menu-follow">
-    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
-		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="https://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="https://feedburner.google.com/fb/lib/images/icons/feed-icon-12x12-orange.gif" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
       <div><a href="http://flattr.com/thing/1191749/PyPy" target="_blank">
-      <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
+      <img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></div>
     </div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />

From pypy.commits at gmail.com  Tue Feb 14 01:00:22 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Mon, 13 Feb 2017 22:00:22 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: merged upstream
Message-ID: <58a29cf6.0a081c0a.33804.9b78@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r860:d1219700aa34
Date: 2017-02-13 21:59 -0800
http://bitbucket.org/pypy/pypy.org/changeset/d1219700aa34/

Log:	merged upstream

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66596 of $105000 (63.4%)
+   $66620 of $105000 (63.4%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2422 left
+    we have $2444 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Tue Feb 14 04:13:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 01:13:49 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58a2ca4d.8e121c0a.81a98.1eb7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r861:90149bca1ec5
Date: 2017-02-14 10:13 +0100
http://bitbucket.org/pypy/pypy.org/changeset/90149bca1ec5/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 63.4
+      value: 63.5
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66620 of $105000 (63.4%)
+   $66625 of $105000 (63.5%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2444 left
+    we have $2448 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Tue Feb 14 05:24:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 02:24:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge fea7fb1fd7fb
Message-ID: <58a2daeb.cd2f1c0a.33734.36be@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90106:af812ba25374
Date: 2017-02-14 11:24 +0100
http://bitbucket.org/pypy/pypy/changeset/af812ba25374/

Log:	hg merge fea7fb1fd7fb

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -71,12 +71,12 @@
 
 def descr_itemsize(space, self):
     assert isinstance(self, W_ArrayBase)
-    return space.wrap(self.itemsize)
+    return space.newint(self.itemsize)
 
 
 def descr_typecode(space, self):
     assert isinstance(self, W_ArrayBase)
-    return space.wrap(self.typecode)
+    return space.newtext(self.typecode)
 
 arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens=['comp_func'],
                               reds='auto')
@@ -284,7 +284,7 @@
             w_item = self.w_getitem(space, i)
             if space.is_true(space.eq(w_item, w_val)):
                 cnt += 1
-        return space.wrap(cnt)
+        return space.newint(cnt)
 
     def descr_index(self, space, w_x):
         """ index(x)
@@ -294,7 +294,7 @@
         for i in range(self.len):
             w_item = self.w_getitem(space, i)
             if space.is_true(space.eq(w_item, w_x)):
-                return space.wrap(i)
+                return space.newint(i)
         raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
     def descr_reverse(self, space):
@@ -361,7 +361,7 @@
         This method is deprecated. Use tobytes instead.
         """
         msg = "tostring() is deprecated. Use tobytes() instead."
-        space.warn(space.wrap(msg), space.w_DeprecationWarning)
+        space.warn(space.newtext(msg), space.w_DeprecationWarning)
         return self.descr_tobytes(space)
 
     def descr_tobytes(self, space):
@@ -392,7 +392,7 @@
                         "array.fromstring(x): x cannot be self")
         s = space.getarg_w('s#', w_s)
         msg = "fromstring() is deprecated. Use frombytes() instead."
-        space.warn(space.wrap(msg), self.space.w_DeprecationWarning)
+        space.warn(space.newtext(msg), self.space.w_DeprecationWarning)
         self._frombytes(space, s)
 
     def descr_frombytes(self, space, w_s):
@@ -433,7 +433,7 @@
             size = ovfcheck(self.itemsize * n)
         except OverflowError:
             raise MemoryError
-        w_item = space.call_method(w_f, 'read', space.wrap(size))
+        w_item = space.call_method(w_f, 'read', space.newint(size))
         item = space.bytes_w(w_item)
         self._frombytes(space, item)
         if len(item) < size:
@@ -477,7 +477,7 @@
         """
         if self.typecode == 'u':
             buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
-            return space.wrap(rffi.wcharpsize2unicode(buf, self.len))
+            return space.newunicode(rffi.wcharpsize2unicode(buf, self.len))
         else:
             raise oefmt(space.w_ValueError,
                         "tounicode() may only be called on type 'u' arrays")
@@ -490,15 +490,15 @@
         The length should be multiplied by the itemsize attribute to calculate
         the buffer length in bytes.
         """
-        w_ptr = space.wrap(self._buffer_as_unsigned())
-        w_len = space.wrap(self.len)
+        w_ptr = space.newint(self._buffer_as_unsigned())
+        w_len = space.newint(self.len)
         return space.newtuple([w_ptr, w_len])
 
     @unwrap_spec(protocol=int)
     def descr_reduce_ex(self, space, protocol):
         """Return state information for pickling."""
         try:
-            w_dict = space.getattr(self, space.wrap('__dict__'))
+            w_dict = space.getattr(self, space.newtext('__dict__'))
         except OperationError:
             w_dict = space.w_None
         from pypy.module.array import reconstructor
@@ -518,7 +518,7 @@
             w_list = self.descr_tolist(space)
             return space.newtuple([
                     space.type(self),
-                    space.newtuple([space.wrap(self.typecode), w_list]),
+                    space.newtuple([space.newtext(self.typecode), w_list]),
                     w_dict])
 
         w_bytes = self.descr_tobytes(space)
@@ -526,8 +526,8 @@
         return space.newtuple([
                 w_array_reconstructor,
                 space.newtuple([space.type(self),
-                                space.wrap(self.typecode),
-                                space.wrap(mformat_code),
+                                space.newtext(self.typecode),
+                                space.newint(mformat_code),
                                 w_bytes]),
                 w_dict])
 
@@ -567,7 +567,7 @@
         self._charbuf_stop()
 
     def descr_len(self, space):
-        return space.wrap(self.len)
+        return space.newint(self.len)
 
     def descr_eq(self, space, w_arr2):
         "x.__eq__(y) <==> x==y"
@@ -733,15 +733,15 @@
 
     def descr_repr(self, space):
         if self.len == 0:
-            return space.wrap("array('%s')" % self.typecode)
+            return space.newtext("array('%s')" % self.typecode)
         elif self.typecode == "u":
             r = space.repr(self.descr_tounicode(space))
             s = u"array('u', %s)" % space.unicode_w(r)
-            return space.wrap(s)
+            return space.newunicode(s)
         else:
             r = space.repr(self.descr_tolist(space))
-            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
-            return space.wrap(s)
+            s = "array('%s', %s)" % (self.typecode, space.text_w(r))
+            return space.newtext(s)
 
 W_ArrayBase.typedef = TypeDef(
     'array.array', None, None, 'read-write',
@@ -807,6 +807,7 @@
         self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
         self.arrayptrtype = lltype.Ptr(self.arraytype)
         self.unwrap, _, self.convert = unwrap.partition('.')
+        assert self.unwrap != 'str_w'
         self.signed = signed
         self.canoverflow = canoverflow
         self.w_class = None
@@ -932,7 +933,8 @@
                         if e.async(space):
                             raise
                         msg = "array item must be " + mytype.unwrap[:-2]
-                        raise OperationError(space.w_TypeError, space.wrap(msg))
+                        raise OperationError(space.w_TypeError,
+                                             space.newtext(msg))
                 else:
                     raise
             if mytype.convert:
@@ -943,7 +945,7 @@
                                 "unsigned %d-byte integer out of range",
                                 mytype.bytes)
                 return rffi.cast(mytype.itemtype, item)
-            if mytype.unwrap == 'str_w' or mytype.unwrap == 'unicode_w':
+            if mytype.unwrap == 'unicode_w':
                 if len(item) != 1:
                     raise oefmt(space.w_TypeError, "array item must be char")
                 item = item[0]
@@ -967,7 +969,7 @@
                     if not mytype.signed:
                         msg = 'un' + msg      # 'signed' => 'unsigned'
                     raise OperationError(self.space.w_OverflowError,
-                                         self.space.wrap(msg))
+                                         self.space.newtext(msg))
             return result
 
         def fromsequence(self, w_seq):
@@ -1046,14 +1048,16 @@
             keepalive_until_here(self)
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
+                return space.newint(item)
             elif mytype.typecode == 'f':
                 item = float(item)
+                return space.newfloat(item)
             elif mytype.typecode == 'u':
                 if ord(item) >= 0x110000:
                     raise oefmt(space.w_ValueError,
                                 "array contains a unicode character out of "
                                 "range(0x110000)")
-            return space.wrap(item)
+            return space.wrap(item) # YYY
 
         # interface
 
@@ -1172,4 +1176,4 @@
     def __init__(self, space):
         w_module = space.getbuiltinmodule('array')
         self.w_array_reconstructor = space.getattr(
-            w_module, space.wrap("_array_reconstructor"))
+            w_module, space.newtext("_array_reconstructor"))
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -213,6 +213,7 @@
         return w_some_obj()
 
     newtext = newutf8
+    newtext_or_none = newutf8
 
     @specialize.argtype(1)
     def wrap(self, x):

From pypy.commits at gmail.com  Tue Feb 14 05:28:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 02:28:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge e725a8848ced
Message-ID: <58a2dbb9.53871c0a.a5ca.37bd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90107:ebb8f9173578
Date: 2017-02-14 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/ebb8f9173578/

Log:	hg merge e725a8848ced

diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py
--- a/pypy/module/_pickle_support/maker.py
+++ b/pypy/module/_pickle_support/maker.py
@@ -15,7 +15,7 @@
 #      Stackless does use this so it might be needed here as well.
 
 def cell_new(space):
-    return space.wrap(instantiate(Cell))
+    return instantiate(Cell)
 
 def code_new(space, __args__):
     w_type = space.gettypeobject(PyCode.typedef)
@@ -24,11 +24,11 @@
 def func_new(space):
     fu = instantiate(Function)
     fu.w_func_dict = space.newdict()
-    return space.wrap(fu)
+    return fu
 
 def module_new(space, w_name, w_dict):
     new_mod = Module(space, w_name, w_dict)
-    return space.wrap(new_mod)
+    return new_mod
 
 def method_new(space, __args__):
     w_type = space.gettypeobject(Method.typedef)
@@ -44,30 +44,30 @@
 
 def frame_new(space):
     new_frame = instantiate(space.FrameClass)   # XXX fish
-    return space.wrap(new_frame)
+    return new_frame
 
 def traceback_new(space):
     tb = instantiate(PyTraceback)
-    return space.wrap(tb)
+    return tb
 
 def generator_new(space):
     new_generator = instantiate(GeneratorIterator)
-    return space.wrap(new_generator)
+    return new_generator
 
 def coroutine_new(space):
     new_coroutine = instantiate(Coroutine)
-    return space.wrap(new_coroutine)
+    return new_coroutine
 
 def longrangeiter_new(space, w_start, w_step, w_len, w_index):
     from pypy.module.__builtin__.functional import W_LongRangeIterator
     new_iter = W_LongRangeIterator(space, w_start, w_step, w_len, w_index)
-    return space.wrap(new_iter)
+    return new_iter
 
 @unwrap_spec(current=int, remaining=int, step=int)
 def intrangeiter_new(space, current, remaining, step):
     from pypy.module.__builtin__.functional import W_IntRangeIterator
     new_iter = W_IntRangeIterator(space, current, remaining, step)
-    return space.wrap(new_iter)
+    return new_iter
 
 @unwrap_spec(identifier=str)
 def builtin_code(space, identifier):
@@ -99,13 +99,12 @@
     a tuple with the positions of NULLs as first element.
     """
     tup = [None] * (len(seq_w) + 1)
-    w = space.wrap
     num = 1
     nulls = [None for i in seq_w if i is None]
     null_num = 0
     for w_obj in seq_w:
         if w_obj is None:
-            nulls[null_num] = w(num - 1)
+            nulls[null_num] = space.newint(num - 1)
             null_num += 1
             w_obj = space.w_None
         tup[num] = w_obj
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -148,7 +148,7 @@
         if isinstance(value, str):
             self.result_w.append(self.space.newbytes(value))
         else:
-            self.result_w.append(self.space.wrap(value))
+            self.result_w.append(self.space.wrap(value)) # YYY
 
     def get_pos(self):
         return self.pos
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -27,15 +27,15 @@
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
-        raise OperationError(space.w_OverflowError, space.wrap(e.msg))
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
-        raise OperationError(get_error(space), space.wrap(e.msg))
+        raise OperationError(get_error(space), space.newtext(e.msg))
     return fmtiter.totalsize
 
 
 @unwrap_spec(format=str)
 def calcsize(space, format):
-    return space.wrap(_calcsize(space, format))
+    return space.newint(_calcsize(space, format))
 
 
 def _pack(space, format, args_w):
@@ -47,9 +47,9 @@
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
-        raise OperationError(space.w_OverflowError, space.wrap(e.msg))
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
-        raise OperationError(get_error(space), space.wrap(e.msg))
+        raise OperationError(get_error(space), space.newtext(e.msg))
     return fmtiter.result.build()
 
 
@@ -78,9 +78,9 @@
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
-        raise OperationError(space.w_OverflowError, space.wrap(e.msg))
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
-        raise OperationError(get_error(space), space.wrap(e.msg))
+        raise OperationError(get_error(space), space.newtext(e.msg))
     return space.newtuple(fmtiter.result_w[:])
 
 

From pypy.commits at gmail.com  Tue Feb 14 06:36:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 03:36:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 22c094827f52
Message-ID: <58a2ebc8.84301c0a.978cc.d3ec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90109:e3793974f52a
Date: 2017-02-14 12:35 +0100
http://bitbucket.org/pypy/pypy/changeset/e3793974f52a/

Log:	hg merge 22c094827f52

diff too long, truncating to 2000 out of 3536 lines

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -742,7 +742,7 @@
         return self.sig
 
     def getdocstring(self, space):
-        return space.newtext(self.docstring)
+        return space.newtext_or_none(self.docstring)
 
     def funcrun(self, func, args):
         return BuiltinCode.funcrun_obj(self, func, None, args)
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -42,12 +42,12 @@
             if strtype is str:
                 return space.newbytes(s)
             else:
-                return space.wrap(s)
+                return space.newunicode(s)
 
         def descr_len(self, space):
             if self.builder is None:
                 raise oefmt(space.w_ValueError, "no length of built builder")
-            return space.wrap(self.builder.getlength())
+            return space.newint(self.builder.getlength())
 
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
diff --git a/pypy/module/__pypy__/interp_identitydict.py b/pypy/module/__pypy__/interp_identitydict.py
--- a/pypy/module/__pypy__/interp_identitydict.py
+++ b/pypy/module/__pypy__/interp_identitydict.py
@@ -11,13 +11,13 @@
     def descr_new(space, w_subtype):
         self = space.allocate_instance(W_IdentityDict, w_subtype)
         W_IdentityDict.__init__(self, space)
-        return space.wrap(self)
+        return self
 
     def descr_len(self, space):
-        return space.wrap(len(self.dict))
+        return space.newint(len(self.dict))
 
     def descr_contains(self, space, w_key):
-        return space.wrap(w_key in self.dict)
+        return space.newbool(w_key in self.dict)
 
     def descr_setitem(self, space, w_key, w_value):
         self.dict[w_key] = w_value
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -8,34 +8,34 @@
 
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
-    return space.wrap(llop.int_add(lltype.Signed, n, m))
+    return space.newint(llop.int_add(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_sub(space, n, m):
-    return space.wrap(llop.int_sub(lltype.Signed, n, m))
+    return space.newint(llop.int_sub(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mul(space, n, m):
-    return space.wrap(llop.int_mul(lltype.Signed, n, m))
+    return space.newint(llop.int_mul(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(int_c_div(n, m))
+    return space.newint(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(int_c_mod(n, m))
+    return space.newint(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
-    return space.wrap(llop.int_lshift(lltype.Signed, n, m))
+    return space.newint(llop.int_lshift(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_rshift(space, n, m):
-    return space.wrap(llop.int_rshift(lltype.Signed, n, m))
+    return space.newint(llop.int_rshift(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def uint_rshift(space, n, m):
     n = r_uint(n)
     x = llop.uint_rshift(lltype.Unsigned, n, m)
-    return space.wrap(intmask(x))
+    return space.newint(intmask(x))
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -13,7 +13,7 @@
 
 
 def internal_repr(space, w_object):
-    return space.wrap('%r' % (w_object,))
+    return space.newtext('%r' % (w_object,))
 
 
 def attach_gdb(space):
@@ -59,7 +59,7 @@
     from pypy.interpreter.function import Function, BuiltinFunction
     func = space.interp_w(Function, w_func)
     bltn = BuiltinFunction(func)
-    return space.wrap(bltn)
+    return bltn
 
 def hidden_applevel(space, w_func):
     """Decorator that hides a function's frame from app-level"""
@@ -79,7 +79,7 @@
 def do_what_I_mean(space, w_crash=None):
     if not space.is_none(w_crash):
         raise ValueError    # RPython-level, uncaught
-    return space.wrap(42)
+    return space.newint(42)
 
 
 def strategy(space, w_obj):
@@ -95,7 +95,7 @@
         name = w_obj.strategy.__class__.__name__
     else:
         raise oefmt(space.w_TypeError, "expecting dict or list or set object")
-    return space.wrap(name)
+    return space.newtext(name)
 
 
 @unwrap_spec(fd='c_int')
@@ -120,8 +120,8 @@
     debug = bool(debug)
     space.sys.debug = debug
     space.setitem(space.builtin.w_dict,
-                  space.wrap('__debug__'),
-                  space.wrap(debug))
+                  space.newtext('__debug__'),
+                  space.newbool(debug))
 
 @unwrap_spec(estimate=int)
 def add_memory_pressure(estimate):
@@ -179,7 +179,7 @@
 
 def stack_almost_full(space):
     """Return True if the stack is more than 15/16th full."""
-    return space.wrap(rstack.stack_almost_full())
+    return space.newbool(rstack.stack_almost_full())
 
 def fsencode(space, w_obj):
     """Direct access to the interp-level fsencode()"""
diff --git a/pypy/module/__pypy__/interp_os.py b/pypy/module/__pypy__/interp_os.py
--- a/pypy/module/__pypy__/interp_os.py
+++ b/pypy/module/__pypy__/interp_os.py
@@ -6,4 +6,4 @@
 @unwrap_spec(name='str0')
 def real_getenv(space, name):
     """Get an OS environment value skipping Python cache"""
-    return space.wrap(os.environ.get(name))
+    return space.newtext_or_none(os.environ.get(name))
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -18,7 +18,7 @@
                 raise exception_from_saved_errno(space, space.w_IOError)
             t = (float(rffi.getintfield(tp, 'c_tv_sec')) +
                  float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001)
-        return space.wrap(t)
+        return space.newfloat(t)
 
     @unwrap_spec(clk_id="c_int")
     def clock_getres(space, clk_id):
@@ -28,4 +28,4 @@
                 raise exception_from_saved_errno(space, space.w_IOError)
             t = (float(rffi.getintfield(tp, 'c_tv_sec')) +
                  float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001)
-        return space.wrap(t)
+        return space.newfloat(t)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -46,19 +46,19 @@
                 w_input = space.newunicode(input)
             w_exc =  space.call_function(
                 w_cls,
-                space.wrap(encoding),
+                space.newtext(encoding),
                 w_input,
-                space.wrap(startpos),
-                space.wrap(endpos),
-                space.wrap(reason))
+                space.newint(startpos),
+                space.newint(endpos),
+                space.newtext(reason))
             w_res = space.call_function(w_errorhandler, w_exc)
             if (not space.isinstance_w(w_res, space.w_tuple)
                 or space.len_w(w_res) != 2
                 or not (space.isinstance_w(
-                            space.getitem(w_res, space.wrap(0)),
+                            space.getitem(w_res, space.newint(0)),
                             space.w_unicode) or
                         (not decode and space.isinstance_w(
-                            space.getitem(w_res, space.wrap(0)),
+                            space.getitem(w_res, space.newint(0)),
                             space.w_bytes)))):
                 if decode:
                     msg = ("decoding error handler must return "
@@ -66,7 +66,7 @@
                 else:
                     msg = ("encoding error handler must return "
                            "(str/bytes, int) tuple")
-                raise OperationError(space.w_TypeError, space.wrap(msg))
+                raise OperationError(space.w_TypeError, space.newtext(msg))
 
             w_replace, w_newpos = space.fixedview(w_res, 2)
             try:
@@ -110,7 +110,7 @@
             return self.unicodedata_handler
         try:
             w_unicodedata = space.getbuiltinmodule("unicodedata")
-            w_getcode = space.getattr(w_unicodedata, space.wrap("_get_code"))
+            w_getcode = space.getattr(w_unicodedata, space.newtext("_get_code"))
         except OperationError:
             return None
         else:
@@ -167,13 +167,13 @@
         # registers new codecs.
         # This import uses the "builtin" import method, and is needed
         # to bootstrap the full importlib module.
-        w_import = space.getattr(space.builtin, space.wrap("__import__"))
-        space.call_function(w_import, space.wrap("encodings"))
+        w_import = space.getattr(space.builtin, space.newtext("__import__"))
+        space.call_function(w_import, space.newtext("encodings"))
         from pypy.module.sys.interp_encoding import base_encoding
         # May be 'utf-8'
         normalized_base = base_encoding.replace("-", "_").lower()
-        space.call_function(w_import, space.wrap("encodings." +
-                                                 normalized_base))
+        space.call_function(w_import, space.newtext("encodings." +
+                                                    normalized_base))
         state.codec_need_encodings = False
         if len(state.codec_search_path) == 0:
             raise oefmt(space.w_LookupError,
@@ -181,7 +181,7 @@
                         "encoding")
     for w_search in state.codec_search_path:
         w_result = space.call_function(w_search,
-                                       space.wrap(normalized_encoding))
+                                       space.newtext(normalized_encoding))
         if not space.is_w(w_result, space.w_None):
             if not (space.isinstance_w(w_result, space.w_tuple) and
                     space.len_w(w_result) == 4):
@@ -198,9 +198,9 @@
 
 def check_exception(space, w_exc):
     try:
-        w_start = space.getattr(w_exc, space.wrap('start'))
-        w_end = space.getattr(w_exc, space.wrap('end'))
-        w_obj = space.getattr(w_exc, space.wrap('object'))
+        w_start = space.getattr(w_exc, space.newtext('start'))
+        w_end = space.getattr(w_exc, space.newtext('end'))
+        w_obj = space.getattr(w_exc, space.newtext('object'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -220,23 +220,23 @@
 
 def ignore_errors(space, w_exc):
     check_exception(space, w_exc)
-    w_end = space.getattr(w_exc, space.wrap('end'))
-    return space.newtuple([space.wrap(u''), w_end])
+    w_end = space.getattr(w_exc, space.newtext('end'))
+    return space.newtuple([space.newunicode(u''), w_end])
 
 def replace_errors(space, w_exc):
     check_exception(space, w_exc)
-    w_start = space.getattr(w_exc, space.wrap('start'))
-    w_end = space.getattr(w_exc, space.wrap('end'))
+    w_start = space.getattr(w_exc, space.newtext('start'))
+    w_end = space.getattr(w_exc, space.newtext('end'))
     size = space.int_w(w_end) - space.int_w(w_start)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
         text = u'?' * size
-        return space.newtuple([space.wrap(text), w_end])
+        return space.newtuple([space.newunicode(text), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
         text = u'\ufffd'
-        return space.newtuple([space.wrap(text), w_end])
+        return space.newtuple([space.newunicode(text), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeTranslateError):
         text = u'\ufffd' * size
-        return space.newtuple([space.wrap(text), w_end])
+        return space.newtuple([space.newunicode(text), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -244,9 +244,9 @@
 def xmlcharrefreplace_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
-        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
+        obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
         builder = UnicodeBuilder()
         pos = start
@@ -262,7 +262,7 @@
             builder.append(unicode(str(code)))
             builder.append(u";")
             pos += 1
-        return space.newtuple([space.wrap(builder.build()), w_end])
+        return space.newtuple([space.newunicode(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -271,9 +271,9 @@
     check_exception(space, w_exc)
     if (space.isinstance_w(w_exc, space.w_UnicodeEncodeError) or
         space.isinstance_w(w_exc, space.w_UnicodeTranslateError)):
-        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
+        obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
         builder = UnicodeBuilder()
         pos = start
@@ -281,11 +281,11 @@
             oc = ord(obj[pos])
             raw_unicode_escape_helper_unicode(builder, oc)
             pos += 1
-        return space.newtuple([space.wrap(builder.build()), w_end])
+        return space.newtuple([space.newunicode(builder.build()), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
-        obj = space.bytes_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
+        obj = space.bytes_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
         builder = UnicodeBuilder()
         pos = start
@@ -293,7 +293,7 @@
             oc = ord(obj[pos])
             raw_unicode_escape_helper_unicode(builder, oc)
             pos += 1
-        return space.newtuple([space.wrap(builder.build()), w_end])
+        return space.newtuple([space.newunicode(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -301,9 +301,9 @@
 def namereplace_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
-        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
+        obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
         builder = UnicodeBuilder()
         pos = start
@@ -318,7 +318,7 @@
                 builder.append(unicode(name))
                 builder.append(u'}')
             pos += 1
-        return space.newtuple([space.wrap(builder.build()), w_end])
+        return space.newtuple([space.newunicode(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -362,10 +362,10 @@
 def surrogatepass_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
-        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
-        encoding = space.str_w(space.getattr(w_exc, space.wrap('encoding')))
+        obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
+        encoding = space.str_w(space.getattr(w_exc, space.newtext('encoding')))
         bytelength, code = get_standard_encoding(encoding)
         if code == ENC_UNKNOWN:
             # Not supported, fail with original exception
@@ -401,9 +401,9 @@
                 builder.append(chr(ch & 0xff))
         return space.newtuple([space.newbytes(builder.build()), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        obj = space.bytes_w(space.getattr(w_exc, space.wrap('object')))
-        encoding = space.str_w(space.getattr(w_exc, space.wrap('encoding')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        obj = space.bytes_w(space.getattr(w_exc, space.newtext('object')))
+        encoding = space.str_w(space.getattr(w_exc, space.newtext('encoding')))
         bytelength, code = get_standard_encoding(encoding)
         ch = 0
         # Try decoding a single surrogate character. If there are more,
@@ -432,8 +432,8 @@
             ch = 0
         if ch == 0:
             raise OperationError(space.type(w_exc), w_exc)
-        return space.newtuple([space.wrap(unichr(ch)),
-                               space.wrap(start + bytelength)])
+        return space.newtuple([space.newunicode(unichr(ch)),
+                               space.newint(start + bytelength)])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -441,9 +441,9 @@
 def surrogateescape_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
-        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
+        obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
         res = ''
         pos = start
@@ -457,9 +457,9 @@
         return space.newtuple([space.newbytes(res), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
         consumed = 0
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        end = space.int_w(space.getattr(w_exc, space.wrap('end')))
-        obj = space.bytes_w(space.getattr(w_exc, space.wrap('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        end = space.int_w(space.getattr(w_exc, space.newtext('end')))
+        obj = space.bytes_w(space.getattr(w_exc, space.newtext('object')))
         replace = u''
         while consumed < 4 and consumed < end - start:
             c = ord(obj[start+consumed])
@@ -471,8 +471,8 @@
         if not consumed:
             # codec complained about ASCII byte.
             raise OperationError(space.type(w_exc), w_exc)
-        return space.newtuple([space.wrap(replace),
-                               space.wrap(start + consumed)])
+        return space.newtuple([space.newunicode(replace),
+                               space.newint(start + consumed)])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -495,13 +495,13 @@
 
 def _call_codec(space, w_decoder, w_obj, action, encoding, errors):
     try:
-        w_res = space.call_function(w_decoder, w_obj, space.wrap(errors))
+        w_res = space.call_function(w_decoder, w_obj, space.newtext(errors))
     except OperationError as operr:
         raise _wrap_codec_error(space, operr, action, encoding)
     if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2):
         raise oefmt(space.w_TypeError,
                     "encoder must return a tuple (object, integer)")
-    return space.getitem(w_res, space.wrap(0))
+    return space.getitem(w_res, space.newint(0))
 
 @unwrap_spec(errors=str)
 def lookup_error(space, errors):
@@ -535,13 +535,13 @@
         encoding = space.sys.defaultencoding
     else:
         encoding = space.str_w(w_encoding)
-    w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
+    w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
     return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
 
 @unwrap_spec(errors='str_or_None')
 def readbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('s#', w_data)
-    return space.newtuple([space.newbytes(s), space.wrap(len(s))])
+    return space.newtuple([space.newbytes(s), space.newint(len(s))])
 
 @unwrap_spec(errors=str)
 def decode(space, w_obj, w_encoding=None, errors='strict'):
@@ -558,7 +558,7 @@
         encoding = space.sys.defaultencoding
     else:
         encoding = space.str_w(w_encoding)
-    w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1))
+    w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1))
     return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
 
 @unwrap_spec(errors=str)
@@ -583,7 +583,7 @@
     codec_info = lookup_codec(space, encoding)
     try:
         is_text_encoding = space.is_true(
-                space.getattr(codec_info, space.wrap('_is_text_encoding')))
+                space.getattr(codec_info, space.newtext('_is_text_encoding')))
     except OperationError as e:
         if e.match(space, space.w_AttributeError):
             is_text_encoding = True
@@ -599,14 +599,14 @@
     if errors is None:
         errors = 'strict'
     w_encoder = space.getitem(
-        lookup_text_codec(space, "codecs.encode()", encoding), space.wrap(0))
+        lookup_text_codec(space, "codecs.encode()", encoding), space.newint(0))
     return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
 
 def decode_text(space, w_obj, encoding, errors):
     if errors is None:
         errors = 'strict'
     w_decoder = space.getitem(
-        lookup_text_codec(space, "codecs.decode()", encoding), space.wrap(1))
+        lookup_text_codec(space, "codecs.decode()", encoding), space.newint(1))
     return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
 
 # ____________________________________________________________
@@ -624,7 +624,7 @@
         state = space.fromcache(CodecState)
         func = getattr(runicode, rname)
         result = func(uni, len(uni), errors, state.encode_error_handler)
-        return space.newtuple([space.newbytes(result), space.wrap(len(uni))])
+        return space.newtuple([space.newbytes(result), space.newint(len(uni))])
     wrap_encoder.func_name = rname
     globals()[name] = wrap_encoder
 
@@ -641,7 +641,7 @@
         func = getattr(runicode, rname)
         result, consumed = func(string, len(string), errors,
                                 final, state.decode_error_handler)
-        return space.newtuple([space.wrap(result), space.wrap(consumed)])
+        return space.newtuple([space.newunicode(result), space.newint(consumed)])
     wrap_decoder.func_name = rname
     globals()[name] = wrap_decoder
 
@@ -684,7 +684,7 @@
         result = runicode.unicode_encode_mbcs(
             uni, len(uni), errors, state.encode_error_handler,
             force_replace=False)
-        return space.newtuple([space.newbytes(result), space.wrap(len(uni))])
+        return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
     @unwrap_spec(string='bufferstr', errors='str_or_None',
                  w_final=WrappedDefault(False))
@@ -697,7 +697,7 @@
             string, len(string), errors,
             final, state.decode_error_handler,
             force_ignore=False)
-        return space.newtuple([space.wrap(result), space.wrap(consumed)])
+        return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=False"
@@ -709,7 +709,7 @@
     result = runicode.unicode_encode_utf_8(
         uni, len(uni), errors, state.encode_error_handler,
         allow_surrogates=False)
-    return space.newtuple([space.newbytes(result), space.wrap(len(uni))])
+    return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
 @unwrap_spec(string='bufferstr', errors='str_or_None',
              w_final = WrappedDefault(False))
@@ -722,7 +722,7 @@
         string, len(string), errors,
         final, state.decode_error_handler,
         allow_surrogates=False)
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
@@ -742,8 +742,8 @@
         consumed = 0
     res, consumed, byteorder = runicode.str_decode_utf_16_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
-    return space.newtuple([space.wrap(res), space.wrap(consumed),
-                           space.wrap(byteorder)])
+    return space.newtuple([space.newunicode(res), space.newint(consumed),
+                           space.newtext(byteorder)])
 
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
@@ -761,8 +761,8 @@
         consumed = 0
     res, consumed, byteorder = runicode.str_decode_utf_32_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
-    return space.newtuple([space.wrap(res), space.wrap(consumed),
-                           space.wrap(byteorder)])
+    return space.newtuple([space.newunicode(res), space.newint(consumed),
+                           space.newtext(byteorder)])
 
 # ____________________________________________________________
 # Charmap
@@ -848,7 +848,7 @@
     if errors is None:
         errors = 'strict'
     if len(string) == 0:
-        return space.newtuple([space.wrap(u''), space.wrap(0)])
+        return space.newtuple([space.newunicode(u''), space.newint(0)])
 
     if space.is_none(w_mapping):
         mapping = None
@@ -860,7 +860,7 @@
     result, consumed = runicode.str_decode_charmap(
         string, len(string), errors,
         final, state.decode_error_handler, mapping)
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 @unwrap_spec(uni=unicode, errors='str_or_None')
 def charmap_encode(space, uni, errors="strict", w_mapping=None):
@@ -875,7 +875,7 @@
     result = runicode.unicode_encode_charmap(
         uni, len(uni), errors,
         state.encode_error_handler, mapping)
-    return space.newtuple([space.newbytes(result), space.wrap(len(uni))])
+    return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
 
 @unwrap_spec(chars=unicode)
@@ -898,7 +898,7 @@
     def call(self, name):
         space = self.space
         try:
-            w_code = space.call_function(self.w_getcode, space.wrap(name))
+            w_code = space.call_function(self.w_getcode, space.newtext(name))
         except OperationError as e:
             if not e.match(space, space.w_KeyError):
                 raise
@@ -920,7 +920,7 @@
         final, state.decode_error_handler,
         unicode_name_handler)
 
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # ____________________________________________________________
 # Raw Unicode escape (accepts bytes or str)
@@ -935,7 +935,7 @@
     result, consumed = runicode.str_decode_raw_unicode_escape(
         string, len(string), errors,
         final, state.decode_error_handler)
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # ____________________________________________________________
 # Unicode-internal
@@ -949,22 +949,22 @@
         return space.newtuple([w_string, space.len(w_string)])
 
     string = space.readbuf_w(w_string).as_str()
-    space.warn(space.wrap("unicode_internal codec has been deprecated"),
+    space.warn(space.newtext("unicode_internal codec has been deprecated"),
                space.w_DeprecationWarning)
 
     if len(string) == 0:
-        return space.newtuple([space.wrap(u''), space.wrap(0)])
+        return space.newtuple([space.newunicode(u''), space.newint(0)])
 
     final = True
     state = space.fromcache(CodecState)
     result, consumed = runicode.str_decode_unicode_internal(
         string, len(string), errors,
         final, state.decode_error_handler)
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 @unwrap_spec(errors='str_or_None')
 def unicode_internal_encode(space, w_uni, errors="strict"):
-    space.warn(space.wrap("unicode_internal codec has been deprecated"),
+    space.warn(space.newtext("unicode_internal codec has been deprecated"),
                space.w_DeprecationWarning)
     if errors is None:
         errors = 'strict'
@@ -973,11 +973,11 @@
         state = space.fromcache(CodecState)
         result = runicode.unicode_encode_unicode_internal(
             uni, len(uni), errors, state.encode_error_handler)
-        return space.newtuple([space.newbytes(result), space.wrap(len(uni))])
+        return space.newtuple([space.newbytes(result), space.newint(len(uni))])
     else:
         # special case for this codec: bytes are returned as is
         string = space.readbuf_w(w_uni).as_str()
-        return space.newtuple([space.newbytes(string), space.wrap(len(string))])
+        return space.newtuple([space.newbytes(string), space.newint(len(string))])
 
 # ____________________________________________________________
 # support for the "string escape" translation
@@ -988,11 +988,11 @@
     data = space.bytes_w(w_data)
     from pypy.objspace.std.bytesobject import string_escape_encode
     result = string_escape_encode(data, False)
-    return space.newtuple([space.newbytes(result), space.wrap(len(data))])
+    return space.newtuple([space.newbytes(result), space.newint(len(data))])
 
 @unwrap_spec(errors='str_or_None')
 def escape_decode(space, w_data, errors='strict'):
     data = space.getarg_w('s#', w_data)
     from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape
     result = PyString_DecodeEscape(space, data, errors, None)
-    return space.newtuple([space.newbytes(result), space.wrap(len(data))])
+    return space.newtuple([space.newbytes(result), space.newint(len(data))])
diff --git a/pypy/module/_collections/__init__.py b/pypy/module/_collections/__init__.py
--- a/pypy/module/_collections/__init__.py
+++ b/pypy/module/_collections/__init__.py
@@ -23,14 +23,14 @@
         # been used...  otherwise, some code is not happy about seeing
         # this code object twice
         space = self.space
-        space.getattr(self, space.wrap('defaultdict'))  # force importing
-        space.delattr(self, space.wrap('__missing__'))
+        space.getattr(self, space.newtext('defaultdict'))  # force importing
+        space.delattr(self, space.newtext('__missing__'))
 
     def startup(self, space):
         # OrderedDict is normally present, but in some cases the line
         # "from __pypy__ import reversed_dict, move_to_end" from
         # _pypy_collections.py raises
-        space.appexec([space.wrap(self)], """(mod):
+        space.appexec([self], """(mod):
             try:
                 from _pypy_collections import OrderedDict
                 mod.OrderedDict = OrderedDict
diff --git a/pypy/module/_collections/interp_defaultdict.py b/pypy/module/_collections/interp_defaultdict.py
--- a/pypy/module/_collections/interp_defaultdict.py
+++ b/pypy/module/_collections/interp_defaultdict.py
@@ -4,7 +4,7 @@
     # An interp-level version of this method.  This is mostly only
     # useful because it can be executed atomically in the presence of
     # threads.
-    w_default_factory = space.getattr(w_self, space.wrap('default_factory'))
+    w_default_factory = space.getattr(w_self, space.newtext('default_factory'))
     if space.is_w(w_default_factory, space.w_None):
         raise OperationError(space.w_KeyError, space.newtuple([w_key]))
     w_value = space.call_function(w_default_factory)
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -159,13 +159,13 @@
             if index >= BLOCKLEN:
                 block = block.rightlink
                 index = 0
-        return space.wrap(result)
+        return space.newint(result)
 
     def extend(self, w_iterable):
         "Extend the right side of the deque with elements from the iterable"
         # Handle case where id(deque) == id(iterable)
         space = self.space
-        if space.is_w(space.wrap(self), w_iterable):
+        if space.is_w(self, w_iterable):
             w_iterable = space.call_function(space.w_list, w_iterable)
         #
         w_iter = space.iter(w_iterable)
@@ -188,7 +188,7 @@
 
     def iadd(self, w_iterable):
         self.extend(w_iterable)
-        return self.space.wrap(self)
+        return self
 
     def mul(self, w_int):
         space = self.space
@@ -226,7 +226,7 @@
         "Extend the left side of the deque with elements from the iterable"
         # Handle case where id(deque) == id(iterable)
         space = self.space
-        if space.is_w(space.wrap(self), w_iterable):
+        if space.is_w(self, w_iterable):
             w_iterable = space.call_function(space.w_list, w_iterable)
         #
         w_iter = space.iter(w_iterable)
@@ -402,7 +402,7 @@
         return W_DequeRevIter(self)
 
     def length(self):
-        return self.space.wrap(self.len)
+        return self.space.newint(self.len)
 
     def repr(self):
         space = self.space
@@ -410,14 +410,14 @@
         w_currently_in_repr = ec._py_repr
         if w_currently_in_repr is None:
             w_currently_in_repr = ec._py_repr = space.newdict()
-        return dequerepr(space, w_currently_in_repr, space.wrap(self))
+        return dequerepr(space, w_currently_in_repr, self)
 
     @specialize.arg(2)
     def compare(self, w_other, op):
         space = self.space
         if not isinstance(w_other, W_Deque):
             return space.w_NotImplemented
-        return space.compare_by_iteration(space.wrap(self), w_other, op)
+        return space.compare_by_iteration(self, w_other, op)
 
     def lt(self, w_other):
         return self.compare(w_other, 'lt')
@@ -489,13 +489,13 @@
             return space.call_function(space.type(self), self)
         else:
             return space.call_function(space.type(self), self,
-                                       space.wrap(self.maxlen))
+                                       space.newint(self.maxlen))
 
     def reduce(self):
         "Return state information for pickling."
         space = self.space
         w_type = space.type(self)
-        w_dict = space.findattr(self, space.wrap('__dict__'))
+        w_dict = space.findattr(self, space.newtext('__dict__'))
         w_list = space.call_function(space.w_list, self)
         if w_dict is None:
             if self.maxlen == sys.maxint:
@@ -503,12 +503,12 @@
                     w_type, space.newtuple([w_list])]
             else:
                 result = [
-                    w_type, space.newtuple([w_list, space.wrap(self.maxlen)])]
+                    w_type, space.newtuple([w_list, space.newint(self.maxlen)])]
         else:
             if self.maxlen == sys.maxint:
                 w_len = space.w_None
             else:
-                w_len = space.wrap(self.maxlen)
+                w_len = space.newint(self.maxlen)
             result = [
                 w_type, space.newtuple([w_list, w_len]), w_dict]
         return space.newtuple(result)
@@ -517,7 +517,7 @@
         if self.maxlen == sys.maxint:
             return self.space.w_None
         else:
-            return self.space.wrap(self.maxlen)
+            return self.space.newint(self.maxlen)
 
 
 app = gateway.applevel("""
@@ -619,10 +619,10 @@
         self.block, self.index = self.deque.locate(index)
 
     def iter(self):
-        return self.space.wrap(self)
+        return self
 
     def length(self):
-        return self.space.wrap(self.counter)
+        return self.space.newint(self.counter)
 
     def next(self):
         space = self.space
@@ -687,10 +687,10 @@
         self.block, self.index = self.deque.locate(self.counter - 1)
 
     def iter(self):
-        return self.space.wrap(self)
+        return self
 
     def length(self):
-        return self.space.wrap(self.counter)
+        return self.space.newint(self.counter)
 
     def next(self):
         space = self.space
diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py
--- a/pypy/module/_multibytecodec/interp_multibytecodec.py
+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py
@@ -24,8 +24,8 @@
             raise wrap_unicodedecodeerror(space, e, input, self.name)
         except RuntimeError:
             raise wrap_runtimeerror(space)
-        return space.newtuple([space.wrap(output),
-                               space.wrap(len(input))])
+        return space.newtuple([space.newunicode(output),
+                               space.newint(len(input))])
 
     @unwrap_spec(input=unicode, errors="str_or_None")
     def encode(self, space, input, errors=None):
@@ -41,7 +41,7 @@
         except RuntimeError:
             raise wrap_runtimeerror(space)
         return space.newtuple([space.newbytes(output),
-                               space.wrap(len(input))])
+                               space.newint(len(input))])
 
 
 MultibyteCodec.typedef = TypeDef(
@@ -58,28 +58,28 @@
         codec = c_codecs.getcodec(name)
     except KeyError:
         raise oefmt(space.w_LookupError, "no such codec is supported.")
-    return space.wrap(MultibyteCodec(name, codec))
+    return MultibyteCodec(name, codec)
 
 
 def wrap_unicodedecodeerror(space, e, input, name):
     return OperationError(
         space.w_UnicodeDecodeError,
         space.newtuple([
-            space.wrap(name),
+            space.newtext(name),
             space.newbytes(input),
-            space.wrap(e.start),
-            space.wrap(e.end),
-            space.wrap(e.reason)]))
+            space.newint(e.start),
+            space.newint(e.end),
+            space.newtext(e.reason)]))
 
 def wrap_unicodeencodeerror(space, e, input, name):
     raise OperationError(
         space.w_UnicodeEncodeError,
         space.newtuple([
-            space.wrap(name),
-            space.wrap(input),
-            space.wrap(e.start),
-            space.wrap(e.end),
-            space.wrap(e.reason)]))
+            space.newtext(name),
+            space.newunicode(input),
+            space.newint(e.start),
+            space.newint(e.end),
+            space.newtext(e.reason)]))
 
 def wrap_runtimeerror(space):
     raise oefmt(space.w_RuntimeError, "internal codec error")
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -143,14 +143,14 @@
             self.ll_chars[i+5] == 't' and
             self.ll_chars[i+6] == 'y'):
             self.pos = i+7
-            return self.space.wrap(rfloat.INFINITY * sign)
+            return self.space.newfloat(rfloat.INFINITY * sign)
         raise DecoderError("Error when decoding Infinity at", i)
 
     def decode_nan(self, i):
         if (self.ll_chars[i]   == 'a' and
             self.ll_chars[i+1] == 'N'):
             self.pos = i+2
-            return self.space.wrap(rfloat.NAN)
+            return self.space.newfloat(rfloat.NAN)
         raise DecoderError("Error when decoding NaN at", i)
 
     def decode_numeric(self, i):
@@ -169,7 +169,7 @@
             return self.decode_int_slow(start)
 
         self.pos = i
-        return self.space.wrap(intval)
+        return self.space.newint(intval)
 
     def decode_float(self, i):
         from rpython.rlib import rdtoa
@@ -177,7 +177,7 @@
         floatval = rdtoa.dg_strtod(start, self.end_ptr)
         diff = rffi.cast(rffi.LONG, self.end_ptr[0]) - rffi.cast(rffi.LONG, start)
         self.pos = i + diff
-        return self.space.wrap(floatval)
+        return self.space.newfloat(floatval)
 
     def decode_int_slow(self, i):
         start = i
@@ -187,7 +187,7 @@
             i += 1
         s = self.getslice(start, i)
         self.pos = i
-        return self.space.call_function(self.space.w_int, self.space.wrap(s))
+        return self.space.call_function(self.space.w_int, self.space.newtext(s))
 
     @always_inline
     def parse_integer(self, i):
@@ -309,7 +309,7 @@
                     content_unicode = strslice2unicode_latin1(self.s, start, i-1)
                 self.last_type = TYPE_STRING
                 self.pos = i
-                return self.space.wrap(content_unicode)
+                return self.space.newunicode(content_unicode)
             elif ch == '\\' or ch < '\x20':
                 self.pos = i-1
                 return self.decode_string_escaped(start)
@@ -330,7 +330,7 @@
                     self.space, content_utf8, allow_surrogates=True)
                 self.last_type = TYPE_STRING
                 self.pos = i
-                return self.space.wrap(content_unicode)
+                return self.space.newunicode(content_unicode)
             elif ch == '\\':
                 i = self.decode_escape_sequence(i, builder)
             elif ch < '\x20':
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -25,7 +25,7 @@
     # call unicode.encode(host, 'idna'), and not host.encode('idna') in case
     # type(host) is not unicode.  See also interp_socket.idna_converter()
     return space.bytes_w(space.call_method(space.w_unicode, 'encode',
-                                           w_host, space.wrap('idna')))
+                                           w_host, space.newtext('idna')))
 
 def gethostbyname(space, w_host):
     """gethostbyname(host) -> address
@@ -38,12 +38,12 @@
         ip = addr.get_host()
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(ip)
+    return space.newtext(ip)
 
 def common_wrapgethost(space, (name, aliases, address_list)):
-    aliases = [space.wrap(alias) for alias in aliases]
-    address_list = [space.wrap(addr.get_host()) for addr in address_list]
-    return space.newtuple([space.wrap(name),
+    aliases = [space.newtext(alias) for alias in aliases]
+    address_list = [space.newtext(addr.get_host()) for addr in address_list]
+    return space.newtuple([space.newtext(name),
                            space.newlist(aliases),
                            space.newlist(address_list)])
 
@@ -89,7 +89,7 @@
         port = rsocket.getservbyname(name, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(port)
+    return space.newint(port)
 
 @unwrap_spec(port=int, w_proto = WrappedDefault(None))
 def getservbyport(space, port, w_proto):
@@ -112,7 +112,7 @@
         service = rsocket.getservbyport(port, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(service)
+    return space.newtext(service)
 
 @unwrap_spec(name=str)
 def getprotobyname(space, name):
@@ -124,7 +124,7 @@
         proto = rsocket.getprotobyname(name)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(proto)
+    return space.newint(proto)
 
 @unwrap_spec(flags=int)
 def getnameinfo(space, w_sockaddr, flags):
@@ -132,8 +132,8 @@
 
     Get host and port for a sockaddr."""
     try:
-        host = space.str_w((space.getitem(w_sockaddr, space.wrap(0))))
-        port = str(space.int_w(space.getitem(w_sockaddr, space.wrap(1))))
+        host = space.str_w((space.getitem(w_sockaddr, space.newint(0))))
+        port = str(space.int_w(space.getitem(w_sockaddr, space.newint(1))))
         lst = rsocket.getaddrinfo(host, port, rsocket.AF_UNSPEC,
                                   rsocket.SOCK_DGRAM, 0,
                                   rsocket.AI_NUMERICHOST)
@@ -145,7 +145,7 @@
         host, servport = rsocket.getnameinfo(addr, flags)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.newtuple([space.wrap(host), space.wrap(servport)])
+    return space.newtuple([space.newtext(host), space.newtext(servport)])
 
 @unwrap_spec(fd=int)
 def dup(space, fd):
@@ -153,7 +153,7 @@
         newfd = rsocket.dup(fd, inheritable=False)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(newfd)
+    return space.newint(newfd)
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -172,8 +172,8 @@
     except SocketError as e:
         raise converted_error(space, e)
     return space.newtuple([
-        space.wrap(W_Socket(space, sock1)),
-        space.wrap(W_Socket(space, sock2))
+        W_Socket(space, sock1),
+        W_Socket(space, sock2)
     ])
 
 # The following 4 functions refuse all negative numbers.
@@ -190,7 +190,7 @@
     if x < 0:
         raise oefmt(space.w_OverflowError,
                     "can't convert negative number to unsigned long")
-    return space.wrap(rsocket.ntohs(intmask(x)))
+    return space.newint(rsocket.ntohs(intmask(x)))
 
 @unwrap_spec(x=r_longlong)
 def ntohl(space, x):
@@ -203,7 +203,7 @@
                     "can't convert negative number to unsigned long")
     if x > LONGLONG_UINT32_MAX:
         raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
-    return space.wrap(rsocket.ntohl(r_uint32(x)))
+    return space.newint(rsocket.ntohl(r_uint32(x)))
 
 @unwrap_spec(x="c_int")
 def htons(space, x):
@@ -214,7 +214,7 @@
     if x < 0:
         raise oefmt(space.w_OverflowError,
                     "can't convert negative number to unsigned long")
-    return space.wrap(rsocket.htons(x))
+    return space.newint(rsocket.htons(x))
 
 @unwrap_spec(x=r_longlong)
 def htonl(space, x):
@@ -227,7 +227,7 @@
                     "can't convert negative number to unsigned long")
     if x > LONGLONG_UINT32_MAX:
         raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
-    return space.wrap(rsocket.htonl(r_uint32(x)))
+    return space.newint(rsocket.htonl(r_uint32(x)))
 
 @unwrap_spec(ip=str)
 def inet_aton(space, ip):
@@ -252,7 +252,7 @@
         ip = rsocket.inet_ntoa(packed)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(ip)
+    return space.newtext(ip)
 
 @unwrap_spec(family=int, ip=str)
 def inet_pton(space, family, ip):
@@ -280,7 +280,7 @@
     except ValueError:
         raise oefmt(space.w_ValueError,
                     "invalid length of packed IP address string")
-    return space.wrap(ip)
+    return space.newtext(ip)
 
 @unwrap_spec(family=int, type=int, proto=int, flags=int)
 def getaddrinfo(space, w_host, w_port,
@@ -319,10 +319,10 @@
                                   proto, flags)
     except SocketError as e:
         raise converted_error(space, e)
-    lst1 = [space.newtuple([space.wrap(family),
-                            space.wrap(socktype),
-                            space.wrap(protocol),
-                            space.wrap(canonname),
+    lst1 = [space.newtuple([space.newint(family),
+                            space.newint(socktype),
+                            space.newint(protocol),
+                            space.newtext(canonname),
                             addr_as_object(addr, INVALID_SOCKET, space)]) # -1 as per cpython
             for (family, socktype, protocol, canonname, addr) in lst]
     return space.newlist(lst1)
@@ -337,7 +337,7 @@
     timeout = rsocket.getdefaulttimeout()
     if timeout < 0.0:
         return space.w_None
-    return space.wrap(timeout)
+    return space.newfloat(timeout)
 
 def setdefaulttimeout(space, w_timeout):
     if space.is_w(w_timeout, space.w_None):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -17,23 +17,23 @@
     make_weakref_descr)
 
 
-# XXX Hack to seperate rpython and pypy
+# XXX Hack to separate rpython and pypy
 def addr_as_object(addr, fd, space):
     from rpython.rlib import _rsocket_rffi as _c
     if isinstance(addr, rsocket.INETAddress):
-        return space.newtuple([space.wrap(addr.get_host()),
-                               space.wrap(addr.get_port())])
+        return space.newtuple([space.newtext(addr.get_host()),
+                               space.newint(addr.get_port())])
     elif isinstance(addr, rsocket.INET6Address):
-        return space.newtuple([space.wrap(addr.get_host()),
-                               space.wrap(addr.get_port()),
-                               space.wrap(addr.get_flowinfo()),
-                               space.wrap(addr.get_scope_id())])
+        return space.newtuple([space.newtext(addr.get_host()),
+                               space.newint(addr.get_port()),
+                               space.newint(addr.get_flowinfo()), # YYY
+                               space.newint(addr.get_scope_id())]) # YYY
     elif rsocket.HAS_AF_PACKET and isinstance(addr, rsocket.PacketAddress):
-        return space.newtuple([space.wrap(addr.get_ifname(fd)),
-                               space.wrap(addr.get_protocol()),
-                               space.wrap(addr.get_pkttype()),
-                               space.wrap(addr.get_hatype()),
-                               space.wrap(addr.get_haddr())])
+        return space.newtuple([space.newtext(addr.get_ifname(fd)),
+                               space.newint(addr.get_protocol()),
+                               space.newint(addr.get_pkttype()),
+                               space.newint(addr.get_hatype()),
+                               space.newtext(addr.get_haddr())])
     elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress):
         path = addr.get_path()
         if _c.linux and len(path) > 0 and path[0] == '\x00':
@@ -42,8 +42,8 @@
         else:
             return space.wrap_fsdecoded(path)
     elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress):
-        return space.newtuple([space.wrap(addr.get_pid()),
-                               space.wrap(addr.get_groups())])
+        return space.newtuple([space.newint(addr.get_pid()),
+                               space.newint(addr.get_groups())])
     # If we don't know the address family, don't raise an
     # exception -- return it as a tuple.
     a = addr.lock()
@@ -51,8 +51,8 @@
     datalen = addr.addrlen - rsocket.offsetof(_c.sockaddr, 'c_sa_data')
     rawdata = ''.join([a.c_sa_data[i] for i in range(datalen)])
     addr.unlock()
-    return space.newtuple([space.wrap(family),
-                          space.wrap(rawdata)])
+    return space.newtuple([space.newint(family),
+                           space.newtext(rawdata)])
 
 # XXX Hack to seperate rpython and pypy
 # XXX a bit of code duplication
@@ -180,7 +180,7 @@
 
 # XXX Hack to seperate rpython and pypy
 def ipaddr_from_object(space, w_sockaddr):
-    host = space.str_w(space.getitem(w_sockaddr, space.wrap(0)))
+    host = space.str_w(space.getitem(w_sockaddr, space.newint(0)))
     addr = rsocket.makeipaddr(host)
     fill_from_object(addr, space, w_sockaddr)
     return addr
@@ -222,29 +222,29 @@
                     pass
 
     def get_type_w(self, space):
-        return space.wrap(self.sock.type)
+        return space.newint(self.sock.type)
 
     def get_proto_w(self, space):
-        return space.wrap(self.sock.proto)
+        return space.newint(self.sock.proto)
 
     def get_family_w(self, space):
-        return space.wrap(self.sock.family)
+        return space.newint(self.sock.family)
 
     def _dealloc_warn(self):
         space = self.space
         try:
             msg = (u"unclosed %s" %
-                   space.unicode_w(space.repr(space.wrap(self))))
-            space.warn(space.wrap(msg), space.w_ResourceWarning)
+                   space.unicode_w(space.repr(self)))
+            space.warn(space.newunicode(msg), space.w_ResourceWarning)
         except OperationError as e:
             # Spurious errors can appear at shutdown
             if e.match(space, space.w_Warning):
-                e.write_unraisable(space, '', space.wrap(self))
+                e.write_unraisable(space, '', self)
 
     def descr_repr(self, space):
         fd = intmask(self.sock.fd)  # Force to signed type even on Windows.
-        return space.wrap("<socket object, fd=%d, family=%d,"
-                          " type=%d, proto=%d>" %
+        return space.newtext("<socket object, fd=%d, family=%d,"
+                             " type=%d, proto=%d>" %
                           (fd, self.sock.family,
                            self.sock.type, self.sock.proto))
 
@@ -258,7 +258,7 @@
         while True:
             try:
                 fd, addr = self.sock.accept(inheritable=False)
-                return space.newtuple([space.wrap(fd),
+                return space.newtuple([space.newint(fd),
                                        addr_as_object(addr, fd, space)])
             except SocketError as e:
                 converted_error(space, e, eintr_retry=True)
@@ -325,14 +325,14 @@
             if error != errno.EINTR:
                 break
             space.getexecutioncontext().checksignals()
-        return space.wrap(error)
+        return space.newint(error)
 
     def fileno_w(self, space):
         """fileno() -> integer
 
         Return the integer file descriptor of the socket.
         """
-        return space.wrap(intmask(self.sock.fd))
+        return space.newint(intmask(self.sock.fd))
 
     def detach_w(self, space):
         """detach()
@@ -341,7 +341,7 @@
         The object cannot be used after this call, but the file descriptor
         can be reused for other purposes.  The file descriptor is returned."""
         fd = self.sock.detach()
-        return space.wrap(intmask(fd))
+        return space.newint(intmask(fd))
 
     def getpeername_w(self, space):
         """getpeername() -> address info
@@ -377,7 +377,7 @@
         """
         if w_buflen is None:
             try:
-                return space.wrap(self.sock.getsockopt_int(level, optname))
+                return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
         buflen = space.int_w(w_buflen)
@@ -392,7 +392,7 @@
         timeout = self.sock.gettimeout()
         if timeout < 0.0:
             return space.w_None
-        return space.wrap(timeout)
+        return space.newfloat(timeout)
 
     @unwrap_spec(backlog="c_int")
     def listen_w(self, space, backlog=min(SOMAXCONN, 128)):
@@ -459,7 +459,7 @@
                 break
             except SocketError as e:
                 converted_error(space, e, eintr_retry=True)
-        return space.wrap(count)
+        return space.newint(count)
 
     @unwrap_spec(data='bufferstr', flags=int)
     def sendall_w(self, space, data, flags=0):
@@ -498,7 +498,7 @@
                 break
             except SocketError as e:
                 converted_error(space, e, eintr_retry=True)
-        return space.wrap(count)
+        return space.newint(count)
 
     @unwrap_spec(flag=int)
     def setblocking_w(self, flag):
@@ -571,7 +571,7 @@
                 break
             except SocketError as e:
                 converted_error(space, e, eintr_retry=True)
-        return space.wrap(nbytes_read)
+        return space.newint(nbytes_read)
 
     @unwrap_spec(nbytes=int, flags=int)
     def recvfrom_into_w(self, space, w_buffer, nbytes=0, flags=0):
@@ -599,7 +599,7 @@
                 raise converted_error(space, e)
         else:
             w_addr = space.w_None
-        return space.newtuple([space.wrap(readlgt), w_addr])
+        return space.newtuple([space.newint(readlgt), w_addr])
 
     @unwrap_spec(cmd=int)
     def ioctl_w(self, space, cmd, w_option):
@@ -638,7 +638,7 @@
                 if value_ptr:
                     lltype.free(value_ptr, flavor='raw')
 
-            return space.wrap(recv_ptr[0])
+            return space.newint(recv_ptr[0]) # YYY
         finally:
             lltype.free(recv_ptr, flavor='raw')
 
@@ -752,10 +752,10 @@
             space.getexecutioncontext().checksignals()
             if eintr_retry:
                 return       # only return None if eintr_retry==True
-        w_exception = space.call_function(w_exception_class, space.wrap(e.errno),
-                                      space.wrap(message))
+        w_exception = space.call_function(w_exception_class, space.newint(e.errno),
+                                      space.newtext(message))
     else:
-        w_exception = space.call_function(w_exception_class, space.wrap(message))
+        w_exception = space.call_function(w_exception_class, space.newtext(message))
     raise OperationError(w_exception_class, w_exception)
 
 # ____________________________________________________________
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -3,14 +3,14 @@
 
 def create_filter(space, w_category, action):
     return space.newtuple([
-        space.wrap(action), space.w_None, w_category,
-        space.w_None, space.wrap(0)])
+        space.newtext(action), space.w_None, w_category,
+        space.w_None, space.newint(0)])
 
 class State:
     def __init__(self, space):
         self.init_filters(space)
         self.w_once_registry = space.newdict()
-        self.w_default_action = space.wrap("default")
+        self.w_default_action = space.newtext("default")
         self.filters_mutated(space)
 
     def filters_mutated(self, space):
@@ -46,14 +46,14 @@
 def get_warnings_attr(space, name):
     try:
         w_module = space.getitem(space.sys.get('modules'),
-                                 space.wrap('warnings'))
+                                 space.newtext('warnings'))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
         return None
 
     try:
-        return space.getattr(w_module, space.wrap(name))
+        return space.getattr(w_module, space.newtext(name))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -122,35 +122,35 @@
 
     # setup registry
     try:
-        w_registry = space.getitem(w_globals, space.wrap("__warningregistry__"))
+        w_registry = space.getitem(w_globals, space.newtext("__warningregistry__"))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
         w_registry = space.newdict()
-        space.setitem(w_globals, space.wrap("__warningregistry__"), w_registry)
+        space.setitem(w_globals, space.newtext("__warningregistry__"), w_registry)
 
     # setup module
     try:
-        w_module = space.getitem(w_globals, space.wrap("__name__"))
+        w_module = space.getitem(w_globals, space.newtext("__name__"))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
-        w_module = space.wrap("<string>")
+        w_module = space.newtext("<string>")
 
     # setup filename
     try:
-        w_filename = space.getitem(w_globals, space.wrap("__file__"))
+        w_filename = space.getitem(w_globals, space.newtext("__file__"))
         filename = space.fsencode_w(w_filename)
     except OperationError as e:
-        if space.str_w(w_module) == '__main__':
+        if space.text_w(w_module) == '__main__':
             w_argv = space.sys.getdictvalue(space, 'argv')
             if w_argv and space.len_w(w_argv) > 0:
-                w_filename = space.getitem(w_argv, space.wrap(0))
+                w_filename = space.getitem(w_argv, space.newint(0))
                 if not space.is_true(w_filename):
-                    w_filename = space.wrap('__main__')
+                    w_filename = space.newtext('__main__')
             else:
                 # embedded interpreters don't have sys.argv
-                w_filename = space.wrap('__main__')
+                w_filename = space.newtext('__main__')
         else:
             w_filename = w_module
     else:
@@ -183,7 +183,7 @@
             check_matched(space, w_mod, w_module) and
             space.abstract_issubclass_w(w_category, w_cat) and
             (ln == 0 or ln == lineno)):
-            return space.str_w(w_action), w_item
+            return space.text_w(w_action), w_item
 
     action = get_default_action(space)
     if not action:
@@ -193,10 +193,10 @@
 def get_default_action(space):
     w_action = get_warnings_attr(space, "defaultaction");
     if w_action is None:
-        return space.str_w(space.fromcache(State).w_default_action)
+        return space.text_w(space.fromcache(State).w_default_action)
 
     space.fromcache(State).w_default_action = w_action
-    return space.str_w(w_action)
+    return space.text_w(w_action)
 
 def get_once_registry(space):
     w_registry = get_warnings_attr(space, "onceregistry");
@@ -228,17 +228,17 @@
 def normalize_module(space, w_filename):
     filename = space.identifier_w(w_filename)
     if len(filename) == 0:
-        return space.wrap("<unknown>")
+        return space.newtext("<unknown>")
     if filename.endswith(".py"):
         n = len(filename) - 3
         assert n >= 0
         filename = filename[:n]
-        return space.newutf8(filename)
+        return space.newtext(filename)
     return w_filename
 
 def show_warning(space, w_filename, lineno, w_text, w_category,
                  w_sourceline=None):
-    w_name = space.getattr(w_category, space.wrap("__name__"))
+    w_name = space.getattr(w_category, space.newtext("__name__"))
     w_stderr = space.sys.get("stderr")
 
     # Print "filename:lineno: category: text\n"
@@ -253,9 +253,9 @@
             # sourceline = linecache.getline(filename, lineno).strip()
             w_builtins = space.getbuiltinmodule('builtins')
             w_linecachemodule = space.call_method(w_builtins, '__import__',
-                                                  space.wrap("linecache"))
+                                                  space.newtext("linecache"))
             w_sourceline = space.call_method(w_linecachemodule, "getline",
-                                             w_filename, space.wrap(lineno))
+                                             w_filename, space.newint(lineno))
             w_sourceline = space.call_method(w_sourceline, "strip")
         except OperationError:
             w_sourceline = None
@@ -298,7 +298,7 @@
         w_text = w_message
         w_message = space.call_function(w_category, w_message)
 
-    w_lineno = space.wrap(lineno)
+    w_lineno = space.newint(lineno)
 
     # create key
     w_key = space.newtuple([w_text, w_category, w_lineno])
@@ -330,7 +330,7 @@
                 warned = update_registry(space, w_registry, w_text, w_category)
         elif action != 'default':
             try:
-                err = space.str_w(space.str(w_item))
+                err = space.text_w(space.str(w_item))
             except OperationError:
                 err = "???"
             raise oefmt(space.w_RuntimeError,
@@ -361,8 +361,8 @@
 
     # Check/get the requisite pieces needed for the loader.
     try:
-        w_loader = space.getitem(w_globals, space.wrap("__loader__"))
-        w_module_name = space.getitem(w_globals, space.wrap("__name__"))
+        w_loader = space.getitem(w_globals, space.newtext("__loader__"))
+        w_module_name = space.getitem(w_globals, space.newtext("__name__"))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
@@ -370,7 +370,7 @@
 
     # Make sure the loader implements the optional get_source() method.
     try:
-        w_get_source = space.getattr(w_loader, space.wrap("get_source"))
+        w_get_source = space.getattr(w_loader, space.newtext("get_source"))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -385,7 +385,7 @@
     w_source_list = space.call_method(w_source, "splitlines")
 
     # Get the source line.
-    w_source_line = space.getitem(w_source_list, space.wrap(lineno - 1))
+    w_source_line = space.getitem(w_source_list, space.newint(lineno - 1))
     return w_source_line
 
 @unwrap_spec(lineno=int, w_module = WrappedDefault(None),
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -292,10 +292,10 @@
     """Return the number of weak references to 'obj'."""
     lifeline = w_obj.getweakref()
     if lifeline is None:
-        return space.wrap(0)
+        return space.newint(0)
     else:
         result = lifeline.traverse(_weakref_count, 0)
-        return space.wrap(result)
+        return space.newint(result)
 
 def _get_weakrefs(lifeline, wref, result):
     w_ref = wref()
diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py
--- a/pypy/module/binascii/interp_hqx.py
+++ b/pypy/module/binascii/interp_hqx.py
@@ -98,7 +98,7 @@
     else:
         if pending_bits > 0:
             raise_Incomplete(space, 'String has incomplete number of bytes')
-    return space.newtuple([space.newbytes(res.build()), space.wrap(done)])
+    return space.newtuple([space.newbytes(res.build()), space.newint(done)])
 
 # ____________________________________________________________
 
@@ -243,4 +243,4 @@
     crc = oldcrc & 0xffff
     for c in data:
         crc = ((crc << 8) & 0xff00) ^ crctab_hqx[((crc >> 8) & 0xff) ^ ord(c)]
-    return space.wrap(crc)
+    return space.newint(crc)
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -262,9 +262,8 @@
 @unwrap_spec(compresslevel=int)
 def descr_compressor__new__(space, w_subtype, compresslevel=9):
     x = space.allocate_instance(W_BZ2Compressor, w_subtype)
-    x = space.interp_w(W_BZ2Compressor, x)
     W_BZ2Compressor.__init__(x, space, compresslevel)
-    return space.wrap(x)
+    return x
 
 class W_BZ2Compressor(W_Root):
     """BZ2Compressor([compresslevel=9]) -> compressor object
@@ -394,9 +393,8 @@
 
 def descr_decompressor__new__(space, w_subtype):
     x = space.allocate_instance(W_BZ2Decompressor, w_subtype)
-    x = space.interp_w(W_BZ2Decompressor, x)
     W_BZ2Decompressor.__init__(x, space)
-    return space.wrap(x)
+    return x
 
 class W_BZ2Decompressor(W_Root):
     """BZ2Decompressor() -> decompressor object
@@ -453,7 +451,7 @@
     def needs_input_w(self, space):
         """ True if more input is needed before more decompressed
             data can be produced. """
-        return space.wrap(self.needs_input)
+        return space.newbool(self.needs_input)
 
     def eof_w(self, space):
         if self.running:
diff --git a/pypy/module/cmath/__init__.py b/pypy/module/cmath/__init__.py
--- a/pypy/module/cmath/__init__.py
+++ b/pypy/module/cmath/__init__.py
@@ -38,8 +38,8 @@
     }
 
     interpleveldefs = {
-        'pi': 'space.wrap(interp_cmath.pi)',
-        'e':  'space.wrap(interp_cmath.e)',
+        'pi': 'space.newfloat(interp_cmath.pi)',
+        'e':  'space.newfloat(interp_cmath.e)',
         'isclose': 'interp_cmath.isclose',
     }
     interpleveldefs.update(dict([(name, 'interp_cmath.wrapped_' + name)
diff --git a/pypy/module/errno/__init__.py b/pypy/module/errno/__init__.py
--- a/pypy/module/errno/__init__.py
+++ b/pypy/module/errno/__init__.py
@@ -22,5 +22,5 @@
 for name in dir(errno):
     if name.startswith('__') or name in Module.interpleveldefs:
         continue
-    Module.interpleveldefs[name] = ("space.wrap(%s)" %
+    Module.interpleveldefs[name] = ("space.newint(%s)" %
                                     (getattr(errno, name), ))
diff --git a/pypy/module/errno/interp_errno.py b/pypy/module/errno/interp_errno.py
--- a/pypy/module/errno/interp_errno.py
+++ b/pypy/module/errno/interp_errno.py
@@ -1,5 +1,7 @@
 import errno
+from rpython.rlib.objectmodel import not_rpython
 
+ at not_rpython
 def get_errorcode(space):
     return space.wrap(errno.errorcode)
 
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -126,7 +126,7 @@
     def descr_str(self, space):
         lgt = len(self.args_w)
         if lgt == 0:
-            return space.wrap('')
+            return space.newtext('')
         elif lgt == 1:
             return space.str(self.args_w[0])
         else:
@@ -137,11 +137,11 @@
         w_base_type = space.gettypeobject(W_BaseException.typedef)
         w_base_str = w_base_type.dict_w["__str__"]
         if not space.is_w(w_str, w_base_str):
-            w_as_str = space.get_and_call_function(w_str, space.wrap(self))
+            w_as_str = space.get_and_call_function(w_str, self)
             return space.call_function(space.w_unicode, w_as_str)
         lgt = len(self.args_w)
         if lgt == 0:
-            return space.wrap(u"")
+            return space.newunicode(u"")
         if lgt == 1:
             return space.call_function(space.w_unicode, self.args_w[0])
         else:
@@ -155,7 +155,7 @@
         else:
             args_repr = u"()"
         clsname = self.getclass(space).getname(space)
-        return space.wrap(clsname + args_repr)
+        return space.newunicode(clsname + args_repr)
 
     def __repr__(self):
         """representation for debugging purposes"""
@@ -199,7 +199,7 @@
         return tb
 
     def descr_getsuppresscontext(self, space):
-        return space.wrap(self.suppress_context)
+        return space.newbool(self.suppress_context)
 
     def descr_setsuppresscontext(self, space, w_value):
         self.suppress_context = space.bool_w(w_value)
@@ -233,7 +233,7 @@
 
     def descr_with_traceback(self, space, w_traceback):
         self.descr_settraceback(space, w_traceback)
-        return space.wrap(self)
+        return self
 
 def _new(cls, basecls=None):
     if basecls is None:
@@ -243,7 +243,7 @@
         exc = space.allocate_instance(cls, w_subtype)
         basecls.__init__(exc, space)
         exc.args_w = args_w
-        return space.wrap(exc)
+        return exc
     descr_new_base_exception.func_name = 'descr_new_' + cls.__name__
     return interp2app(descr_new_base_exception)
 
@@ -368,7 +368,7 @@
                                                  w_end, w_reason])
 
     def descr_str(self, space):
-        return space.appexec([space.wrap(self)], r"""(self):
+        return space.appexec([self], r"""(self):
             if self.object is None:
                 return ""
             if self.end == self.start + 1:
@@ -399,7 +399,7 @@
 
 def key_error_str(self, space):
     if len(self.args_w) == 0:
-        return space.wrap('')
+        return space.newtext('')
     elif len(self.args_w) == 1:
         return space.repr(self.args_w[0])
     else:
@@ -463,10 +463,10 @@
         # (see http://bugs.python.org/issue12555#msg148829 )
         if space.is_w(w_subtype, space.w_OSError):
             return False
-        if ((space.getattr(w_subtype, space.wrap('__init__')) !=
-             space.getattr(space.w_OSError, space.wrap('__init__'))) and
-            (space.getattr(w_subtype, space.wrap('__new__')) ==
-             space.getattr(space.w_OSError, space.wrap('__new__')))):
+        if ((space.getattr(w_subtype, space.newtext('__init__')) !=
+             space.getattr(space.w_OSError, space.newtext('__init__'))) and
+            (space.getattr(w_subtype, space.newtext('__new__')) ==
+             space.getattr(space.w_OSError, space.newtext('__new__')))):
             return True
         return False
 
@@ -515,7 +515,7 @@
         if not W_OSError._use_init(space, w_subtype):
             exc._init_error(space, args_w, w_errno, w_strerror, w_filename,
                             w_filename2)
-        return space.wrap(exc)
+        return exc
 
     def descr_init(self, space, __args__):
         args_w, kwds_w = __args__.unpack()
@@ -573,16 +573,16 @@
             strerror = space.unicode_w(space.str(self.w_strerror))
             if not space.is_w(self.w_filename, space.w_None):
                 if not space.is_w(self.w_filename2, space.w_None):
-                    return space.wrap(u"[Errno %s] %s: %s -> %s" % (
+                    return space.newunicode(u"[Errno %s] %s: %s -> %s" % (
                         errno,
                         strerror,
                         space.unicode_w(space.repr(self.w_filename)),
                         space.unicode_w(space.repr(self.w_filename2))))
-                return space.wrap(u"[Errno %s] %s: %s" % (
+                return space.newunicode(u"[Errno %s] %s: %s" % (
                     errno,
                     strerror,
                     space.unicode_w(space.repr(self.w_filename))))
-            return space.wrap(u"[Errno %s] %s" % (
+            return space.newunicode(u"[Errno %s] %s" % (
                 errno,
                 strerror,
             ))
@@ -591,7 +591,7 @@
     def descr_get_written(self, space):
         if self.written == -1:
             raise oefmt(space.w_AttributeError, "characters_written")
-        return space.wrap(self.written)
+        return space.newint(self.written)
 
     def descr_set_written(self, space, w_written):
         self.written = space.int_w(w_written)
@@ -630,7 +630,7 @@
         else:
             errno = self._winerror_to_errno.get(errno, self._default_errno)
         self.w_winerror = self.w_errno
-        self.w_errno = space.wrap(errno)
+        self.w_errno = space.newint(errno)
 
     def descr_str(self, space):
         if (not space.is_w(self.w_winerror, space.w_None) and
@@ -638,11 +638,11 @@
             winerror = space.int_w(self.w_winerror)
             strerror = space.unicode_w(self.w_strerror)
             if not space.is_w(self.w_filename, space.w_None):
-                return space.wrap(u"[Error %d] %s: %s" % (
+                return space.newunicode(u"[Error %d] %s: %s" % (
                     winerror,
                     strerror,
                     space.unicode_w(self.w_filename)))
-            return space.wrap(u"[Error %d] %s" % (winerror,
+            return space.newunicode(u"[Error %d] %s" % (winerror,
                                                   strerror))
         return W_BaseException.descr_str(self, space)
 
@@ -786,7 +786,7 @@
             args_w = [self.args_w[0], w_tuple]
             args_repr = space.unicode_w(space.repr(space.newtuple(args_w)))
             clsname = self.getclass(space).getname(space)
-            return space.wrap(clsname + args_repr)
+            return space.newtext(clsname + args_repr)
         else:
             return W_Exception.descr_repr(self, space)
 
@@ -817,11 +817,11 @@
             text = text[start:]
         # Check for legacy print statements
         if text.startswith(u"print "):
-            self.w_msg = space.wrap("Missing parentheses in call to 'print'")
+            self.w_msg = space.newtext("Missing parentheses in call to 'print'")
             return True
         # Check for legacy exec statements
         if text.startswith(u"exec "):
-            self.w_msg = space.wrap("Missing parentheses in call to 'exec'")
+            self.w_msg = space.newtext("Missing parentheses in call to 'exec'")
             return True
         return False
 
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -30,7 +30,7 @@
 def wrap(space, gcref):
     w_obj = try_cast_gcref_to_w_root(gcref)
     if w_obj is None:
-        w_obj = space.wrap(W_GcRef(gcref))
+        w_obj = W_GcRef(gcref)
     return w_obj
 
 def unwrap(space, w_obj):
@@ -149,7 +149,7 @@
     size = rgc.get_rpy_memory_usage(gcref)
     if size < 0:
         raise missing_operation(space)
-    return space.wrap(size)
+    return space.newint(size)
 
 def get_rpy_type_index(space, w_obj):
     """Return an integer identifying the RPython type of the given
@@ -159,7 +159,7 @@
     index = rgc.get_rpy_type_index(gcref)
     if index < 0:
         raise missing_operation(space)
-    return space.wrap(index)
+    return space.newint(index)
 
 def get_objects(space):
     """Return a list of all app-level objects."""
@@ -206,5 +206,5 @@
 
 def get_typeids_list(space):
     l = rgc.get_typeids_list()
-    list_w = [space.wrap(l[i]) for i in range(len(l))]
+    list_w = [space.newint(l[i]) for i in range(len(l))]
     return space.newlist(list_w)
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -10,7 +10,7 @@
 def extension_suffixes(space):
     suffixes_w = []
     if 1:   #if space.config.objspace.usemodules.cpyext:
-        suffixes_w.append(space.wrap(importing.get_so_extension(space)))
+        suffixes_w.append(space.newtext(importing.get_so_extension(space)))
     return space.newlist(suffixes_w)
 
 def get_magic(space):
@@ -27,7 +27,7 @@
 def get_tag(space):
     """get_tag() -> string
     Return the magic tag for .pyc or .pyo files."""
-    return space.wrap(importing.PYC_TAG)
+    return space.newtext(importing.PYC_TAG)
 
 def get_file(space, w_file, filename, filemode):
     if space.is_none(w_file):
@@ -48,14 +48,14 @@
 def create_dynamic(space, w_spec, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
-    w_modulename = space.getattr(w_spec, space.wrap("name"))
-    w_path = space.getattr(w_spec, space.wrap("origin"))
+    w_modulename = space.getattr(w_spec, space.newtext("name"))
+    w_path = space.getattr(w_spec, space.newtext("origin"))
     filename = space.fsencode_w(w_path)
     importing.load_c_extension(space, filename, space.str_w(w_modulename))
     return importing.check_sys_modules(space, w_modulename)
 
 def create_builtin(space, w_spec):
-    w_name = space.getattr(w_spec, space.wrap("name"))
+    w_name = space.getattr(w_spec, space.newtext("name"))
     name = space.str0_w(w_name)
     # force_init is needed to make reload actually reload instead of just
     # using the already-present module in sys.modules.
@@ -74,10 +74,10 @@
 def is_builtin(space, w_name):
     name = space.str0_w(w_name)
     if name not in space.builtin_modules:
-        return space.wrap(0)
+        return space.newint(0)
     if space.finditem(space.sys.get('modules'), w_name) is not None:
-        return space.wrap(-1)   # cannot be initialized again
-    return space.wrap(1)
+        return space.newint(-1)   # cannot be initialized again
+    return space.newint(1)
 
 def is_frozen(space, w_name):
     return space.w_False
@@ -94,7 +94,7 @@
 
 def lock_held(space):
     if space.config.objspace.usemodules.thread:
-        return space.wrap(importing.getimportlock(space).lock_held_by_anyone())
+        return space.newbool(importing.getimportlock(space).lock_held_by_anyone())
     else:
         return space.w_False
 
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -13,7 +13,7 @@
         self.w_step = w_step
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         w_c = self.w_c
@@ -23,7 +23,7 @@
     def single_argument(self):
         space = self.space
         return (space.isinstance_w(self.w_step, space.w_int) and
-                space.eq_w(self.w_step, space.wrap(1)))
+                space.eq_w(self.w_step, space.newint(1)))
 
     def repr_w(self):
         space = self.space
@@ -33,7 +33,7 @@
         else:
             step = space.str_w(space.repr(self.w_step))
             s = 'count(%s, %s)' % (c, step)
-        return self.space.wrap(s)
+        return self.space.newtext(s)
 
     def reduce_w(self):
         space = self.space
@@ -55,7 +55,7 @@
     check_number(space, w_step)
     r = space.allocate_instance(W_Count, w_subtype)
     r.__init__(space, w_start, w_step)
-    return space.wrap(r)
+    return r
 
 W_Count.typedef = TypeDef(
         'itertools.count',
@@ -99,12 +99,12 @@
         return self.w_obj

From pypy.commits at gmail.com  Tue Feb 14 06:36:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 03:36:38 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: tweaks
Message-ID: <58a2ebc6.56711c0a.ac977.fc64@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90108:a4d3c68d1442
Date: 2017-02-14 12:09 +0100
http://bitbucket.org/pypy/pypy/changeset/a4d3c68d1442/

Log:	tweaks

diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -12,14 +12,8 @@
         self.lastlineno = lastlineno
 
     def wrap_info(self, space):
-        if self.filename is None:
-            w_filename = space.w_None
-        else:
-            w_filename = space.newtext_or_none(self.filename)
-        if self.text is None:
-            w_text = space.w_None
-        else:
-            w_text = space.newtext(self.text)
+        w_filename = space.newtext_or_none(self.filename)
+        w_text = space.newtext_or_none(self.text)
         return space.newtuple([space.newtext(self.msg),
                                space.newtuple([w_filename,
                                                space.newint(self.lineno),
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -202,7 +202,7 @@
 def get_typeids_z(space):
     a = rgc.get_typeids_z()
     s = ''.join([a[i] for i in range(len(a))])
-    return space.newtext(s)
+    return space.newbytes(s)
 
 def get_typeids_list(space):
     l = rgc.get_typeids_list()

From pypy.commits at gmail.com  Tue Feb 14 10:27:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 07:27:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge ee5ad76cdf97
Message-ID: <58a321d2.04ba1c0a.fa2e2.3c3d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90110:cf2f4a07e7c1
Date: 2017-02-14 15:53 +0100
http://bitbucket.org/pypy/pypy/changeset/cf2f4a07e7c1/

Log:	hg merge ee5ad76cdf97

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -1,3 +1,5 @@
+from rpython.rlib.rarithmetic import (r_uint, r_ulonglong, r_longlong,
+                                      maxint, intmask)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rstring import StringBuilder
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -100,6 +100,7 @@
     w_type = _precheck_for_new(space, w_type)
 
     if _excess_args(__args__):
+        w_parent_new, _ = space.lookup_in_type_where(w_type, '__new__')
         w_parent_init, _ = space.lookup_in_type_where(w_type, '__init__')
         w_parent_new, _ = space.lookup_in_type_where(w_type, '__new__')
         if (w_parent_init is space.w_object or
@@ -118,6 +119,7 @@
 def descr__init__(space, w_obj, __args__):
     if _excess_args(__args__):
         w_type = space.type(w_obj)
+        w_parent_init, _ = space.lookup_in_type_where(w_type, '__init__')
         w_parent_new, _ = space.lookup_in_type_where(w_type, '__new__')
         w_parent_init, _ = space.lookup_in_type_where(w_type, '__init__')
         if (w_parent_new is space.w_object or
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -657,7 +657,7 @@
         prefix = self._op_val(space, w_prefix)
         return endswith(value, prefix, start, end)
 
-    def _strip(self, space, w_chars, left, right):
+    def _strip(self, space, w_chars, left, right, name='strip'):
         "internal function called by str_xstrip methods"
         value = self._val(space)
         chars = self._op_val(space, w_chars)
@@ -697,17 +697,17 @@
     def descr_strip(self, space, w_chars=None):
         if space.is_none(w_chars):
             return self._strip_none(space, left=1, right=1)
-        return self._strip(space, w_chars, left=1, right=1)
+        return self._strip(space, w_chars, left=1, right=1, name='strip')
 
     def descr_lstrip(self, space, w_chars=None):
         if space.is_none(w_chars):
             return self._strip_none(space, left=1, right=0)
-        return self._strip(space, w_chars, left=1, right=0)
+        return self._strip(space, w_chars, left=1, right=0, name='lstrip')
 
     def descr_rstrip(self, space, w_chars=None):
         if space.is_none(w_chars):
             return self._strip_none(space, left=0, right=1)
-        return self._strip(space, w_chars, left=0, right=1)
+        return self._strip(space, w_chars, left=0, right=1, name='rstrip')
 
     def descr_swapcase(self, space):
         selfvalue = self._val(space)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -422,6 +422,9 @@
     def descr_rmod(self, space, w_values):
         return mod_format(space, w_values, self, fmt_type=FORMAT_UNICODE)
 
+    def descr_rmod(self, space, w_values):
+        return mod_format(space, w_values, self, do_unicode=True)
+
     def descr_translate(self, space, w_table):
         selfvalue = self._value
         w_sys = space.getbuiltinmodule('sys')

From pypy.commits at gmail.com  Tue Feb 14 10:27:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 07:27:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge de24352afe1b
Message-ID: <58a321d4.c9bf1c0a.8cd63.8e0b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90111:77a934df9a3a
Date: 2017-02-14 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/77a934df9a3a/

Log:	hg merge de24352afe1b

diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -36,7 +36,7 @@
         w_args, w_kwds = __args__.topacked()
         bottomframe = space.createframe(get_entrypoint_pycode(space),
                                         get_w_module_dict(space), None)
-        bottomframe.locals_cells_stack_w[0] = space.wrap(self)
+        bottomframe.locals_cells_stack_w[0] = self
         bottomframe.locals_cells_stack_w[1] = w_callable
         bottomframe.locals_cells_stack_w[2] = w_args
         bottomframe.locals_cells_stack_w[3] = w_kwds
@@ -135,13 +135,13 @@
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
-    return space.wrap(r)
+    return r
 
 def unpickle(space, w_subtype):
     """Pickle support."""
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
-    return space.wrap(r)
+    return r
 
 
 W_Continulet.typedef = TypeDef(
@@ -165,7 +165,7 @@
     def __init__(self, space):
         self.space = space
         w_module = space.getbuiltinmodule('_continuation')
-        self.w_error = space.getattr(w_module, space.wrap('error'))
+        self.w_error = space.getattr(w_module, space.newtext('error'))
         # the following function switches away immediately, so that
         # continulet.__init__() doesn't immediately run func(), but it
         # also has the hidden purpose of making sure we have a single
@@ -188,7 +188,7 @@
 
 def geterror(space, message):
     cs = space.fromcache(State)
-    return OperationError(cs.w_error, space.wrap(message))
+    return OperationError(cs.w_error, space.newtext(message))
 
 def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
--- a/pypy/module/_continuation/interp_pickle.py
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -24,8 +24,8 @@
     elif self.sthread.is_empty_handle(self.h):
         w_frame = space.w_None
     else:
-        w_frame = space.wrap(self.bottomframe)
-    w_continulet_type = space.type(space.wrap(self))
+        w_frame = self.bottomframe
+    w_continulet_type = space.type(self)
     w_dict = self.getdict(space) or space.w_None
     args = [getunpickle(space),
             space.newtuple([w_continulet_type]),
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -23,15 +23,15 @@
         self.line_num = 0
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         msg = u'line %d: %s' % (self.line_num, msg)
         w_module = space.getbuiltinmodule('_csv')
-        w_error = space.getattr(w_module, space.wrap('Error'))
-        raise OperationError(w_error, space.wrap(msg))
+        w_error = space.getattr(w_module, space.newtext('Error'))
+        raise OperationError(w_error, space.newunicode(msg))
 
     def add_char(self, field_builder, c):
         assert field_builder is not None
@@ -44,9 +44,9 @@
         field = field_builder.build()
         if self.numeric_field:
             self.numeric_field = False
-            w_obj = space.call_function(space.w_float, space.wrap(field))
+            w_obj = space.call_function(space.w_float, space.newtext(field))
         else:
-            w_obj = space.wrap(field)
+            w_obj = space.newtext(field)
         self.fields_w.append(w_obj)
 
     def next_w(self):
@@ -267,4 +267,4 @@
     old_limit = field_limit.limit
     if new_limit >= 0:
         field_limit.limit = new_limit
-    return space.wrap(old_limit)
+    return space.newint(old_limit)
diff --git a/pypy/module/binascii/interp_binascii.py b/pypy/module/binascii/interp_binascii.py
--- a/pypy/module/binascii/interp_binascii.py
+++ b/pypy/module/binascii/interp_binascii.py
@@ -9,16 +9,16 @@
 
 def raise_Error(space, msg):
     w_error = space.fromcache(Cache).w_error
-    raise OperationError(w_error, space.wrap(msg))
+    raise OperationError(w_error, space.newtext(msg))
 
 def raise_Incomplete(space, msg):
     w_error = space.fromcache(Cache).w_incomplete
-    raise OperationError(w_error, space.wrap(msg))
+    raise OperationError(w_error, space.newtext(msg))
 
 # a2b functions accept bytes and buffers, but also ASCII strings.
 class AsciiBufferUnwrapper(Unwrapper):
     def unwrap(self, space, w_value):
         if space.isinstance_w(w_value, space.w_unicode):
-            w_value = space.call_method(w_value, "encode", space.wrap("ascii"))
+            w_value = space.call_method(w_value, "encode", space.newtext("ascii"))
         return space.bufferstr_w(w_value)
 
diff --git a/pypy/module/binascii/interp_crc32.py b/pypy/module/binascii/interp_crc32.py
--- a/pypy/module/binascii/interp_crc32.py
+++ b/pypy/module/binascii/interp_crc32.py
@@ -6,4 +6,4 @@
 def crc32(space, data, oldcrc=0):
     "Compute the CRC-32 incrementally."
     crc = rzipfile.crc32(data, r_uint(oldcrc))
-    return space.wrap(crc)
+    return space.newint(crc)
diff --git a/pypy/module/crypt/interp_crypt.py b/pypy/module/crypt/interp_crypt.py
--- a/pypy/module/crypt/interp_crypt.py
+++ b/pypy/module/crypt/interp_crypt.py
@@ -21,4 +21,4 @@
     if not res:
         return space.w_None
     str_res = rffi.charp2str(res)
-    return space.wrap(str_res)
+    return space.newtext(str_res)
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -153,7 +153,7 @@
 
 def is_enabled(space):
     "is_enabled()->bool: check if the handler is enabled"
-    return space.wrap(space.fromcache(Handler).is_enabled())
+    return space.newbool(space.fromcache(Handler).is_enabled())
 
 @unwrap_spec(all_threads=int)
 def dump_traceback(space, w_file=None, all_threads=0):
@@ -178,7 +178,7 @@
 
 @unwrap_spec(signum=int)
 def unregister(space, signum):
-    return space.wrap(space.fromcache(Handler).unregister(signum))
+    return space.newbool(space.fromcache(Handler).unregister(signum))
 
 
 # for tests...
@@ -206,4 +206,4 @@
 @unwrap_spec(levels=int)
 def stack_overflow(space, levels=100000000):
     levels = float(levels)
-    return space.wrap(cintf.pypy_faulthandler_stackoverflow(levels))
+    return space.newfloat(cintf.pypy_faulthandler_stackoverflow(levels))
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -76,7 +76,7 @@
     def __init__(self, space, w_f):
         AbstractReaderWriter.__init__(self, space)
         try:
-            self.func = space.getattr(w_f, space.wrap('read'))
+            self.func = space.getattr(w_f, space.newtext('read'))
             # XXX how to check if it is callable?
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
@@ -86,7 +86,7 @@
 
     def read(self, n):
         space = self.space
-        w_ret = space.call_function(self.func, space.wrap(n))
+        w_ret = space.call_function(self.func, space.newint(n))
         ret = space.str_w(w_ret)
         if len(ret) < n:
             self.raise_eof()
@@ -101,7 +101,7 @@
 class _Base(object):
     def raise_exc(self, msg):
         space = self.space
-        raise OperationError(space.w_ValueError, space.wrap(msg))
+        raise OperationError(space.w_ValueError, space.newtext(msg))
 
 class Marshaller(_Base):
     """
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -40,9 +40,9 @@
 
 def wrap_uid(space, uid):
     if uid <= r_uint(sys.maxint):
-        return space.wrap(intmask(uid))
+        return space.newint(intmask(uid))
     else:
-        return space.wrap(uid)     # an unsigned number
+        return space.newint(uid)     # an unsigned number
 wrap_gid = wrap_uid
 
 class FileEncoder(object):
@@ -245,7 +245,7 @@
     except OSError as e:
         rposix.c_close(fd)
         raise wrap_oserror2(space, e, w_path, eintr_retry=False)
-    return space.wrap(fd)
+    return space.newint(fd)
 
 @unwrap_spec(fd=c_int, position=r_longlong, how=c_int)
 def lseek(space, fd, position, how):
@@ -257,7 +257,7 @@
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(pos)
+        return space.newint(pos)
 
 @unwrap_spec(fd=c_int)
 def isatty(space, fd):
@@ -268,7 +268,7 @@
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(res)
+        return space.newbool(res)
 
 @unwrap_spec(fd=c_int, length=int)
 def read(space, fd, length):
@@ -292,7 +292,7 @@
         except OSError as e:
             wrap_oserror(space, e, eintr_retry=True)
         else:
-            return space.wrap(res)
+            return space.newint(res)
 
 @unwrap_spec(fd=c_int)
 def close(space, fd):
@@ -391,7 +391,7 @@
         if i < rposix_stat.N_INDEXABLE_FIELDS:
             # get the first 10 items by indexing; this gives us
             # 'st_Xtime' as an integer, too
-            w_value = space.wrap(st[i])
+            w_value = space.newint(st[i])
             lst[i] = w_value
         else:
             try:
@@ -401,8 +401,8 @@
                 assert name.startswith('nsec_')
                 value = rposix_stat.get_stat_ns_as_bigint(st, name[5:])
                 value = value.tolong() % 1000000000
-            w_value = space.wrap(value)
-            space.setitem(w_keywords, space.wrap(name), w_value)
+            w_value = space.newint(value)
+            space.setitem(w_keywords, space.newtext(name), w_value)
 
     # Note: 'w_keywords' contains the three attributes 'nsec_Xtime'.
     # We have an app-level property in app_posix.stat_result to
@@ -411,27 +411,27 @@
     # non-rounded values for name-based access
     if stat_float_times:
         space.setitem(w_keywords,
-                      space.wrap('st_atime'), space.wrap(st.st_atime))
+                      space.newtext('st_atime'), space.newfloat(st.st_atime))
         space.setitem(w_keywords,
-                      space.wrap('st_mtime'), space.wrap(st.st_mtime))
+                      space.newtext('st_mtime'), space.newfloat(st.st_mtime))
         space.setitem(w_keywords,
-                      space.wrap('st_ctime'), space.wrap(st.st_ctime))
+                      space.newtext('st_ctime'), space.newfloat(st.st_ctime))
     #else:
     #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
-                                  space.wrap('stat_result'))
+                                  space.newtext('stat_result'))
     return space.call_function(w_stat_result, w_tuple, w_keywords)
 
 
 def build_statvfs_result(space, st):
     vals_w = [None] * len(rposix_stat.STATVFS_FIELDS)
     for i, (name, _) in STATVFS_FIELDS:
-        vals_w[i] = space.wrap(getattr(st, name))
+        vals_w[i] = space.newint(getattr(st, name))
     w_tuple = space.newtuple(vals_w)
     w_statvfs_result = space.getattr(
-        space.getbuiltinmodule(os.name), space.wrap('statvfs_result'))
+        space.getbuiltinmodule(os.name), space.newtext('statvfs_result'))
     return space.call_function(w_statvfs_result, w_tuple)
 
 
@@ -521,7 +521,7 @@
     state = space.fromcache(StatState)
 
     if newval == -1:
-        return space.wrap(state.stat_float_times)
+        return space.newbool(state.stat_float_times)
     else:
         state.stat_float_times = (newval != 0)
 
@@ -564,7 +564,7 @@
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(newfd)
+        return space.newint(newfd)
 
 @unwrap_spec(fd=c_int, fd2=c_int, inheritable=bool)
 def dup2(space, fd, fd2, inheritable=1):
@@ -621,7 +621,7 @@
     except OSError as e:
         raise wrap_oserror2(space, e, w_path, eintr_retry=False)
     else:
-        return space.wrap(ok)
+        return space.newint(ok)
 
 
 def times(space):
@@ -635,11 +635,11 @@
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.newtuple([space.wrap(times[0]),
-                               space.wrap(times[1]),
-                               space.wrap(times[2]),
-                               space.wrap(times[3]),
-                               space.wrap(times[4])])
+        return space.newtuple([space.newfloat(times[0]),
+                               space.newfloat(times[1]),
+                               space.newfloat(times[2]),
+                               space.newfloat(times[3]),
+                               space.newfloat(times[4])])
 
 @unwrap_spec(command='fsencode')
 def system(space, command):
@@ -649,7 +649,7 @@
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(rc)
+        return space.newint(rc)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
 def unlink(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
@@ -695,7 +695,7 @@
         if space.isinstance_w(w_path, space.w_unicode):
             path = FileEncoder(space, w_path)
             fullpath = rposix.getfullpathname(path)
-            w_fullpath = space.wrap(fullpath)
+            w_fullpath = space.newtext(fullpath)
         else:
             path = space.str0_w(w_path)
             fullpath = rposix.getfullpathname(path)
@@ -722,7 +722,7 @@
         except OSError as e:
             raise wrap_oserror(space, e, eintr_retry=False)
         else:
-            return space.wrap(cur)
+            return space.newunicode(cur)
 else:
     def getcwd(space):
         """Return the current working directory as a string."""
@@ -783,7 +783,7 @@
 def strerror(space, code):
     """Translate an error code to a message string."""
     try:
-        return space.wrap(_strerror(code))
+        return space.newtext(_strerror(code))
     except ValueError:
         raise oefmt(space.w_ValueError, "strerror() argument out of range")
 
@@ -800,7 +800,7 @@
 def getstatfields(space):
     # for app_posix.py: export the list of 'st_xxx' names that we know
     # about at RPython level
-    return space.newlist([space.wrap(name) for _, (name, _) in STAT_FIELDS])
+    return space.newlist([space.newtext(name) for _, (name, _) in STAT_FIELDS])
 
 
 class State:
@@ -832,7 +832,7 @@
         for key, value in rwin32._wenviron_items():
             if isinstance(key, str):
                 key = key.upper()
-            space.setitem(w_env, space.wrap(key), space.wrap(value))
+            space.setitem(w_env, space.newtext(key), space.newtext(value))
 
     @unwrap_spec(name=unicode, value=unicode)
     def putenv(space, name, value):
@@ -913,7 +913,7 @@
     result_w = [None] * len_result
     for i in range(len_result):
         if _WIN32:
-            result_w[i] = space.wrap(result[i])
+            result_w[i] = space.newunicode(result[i])
         else:
             result_w[i] = space.wrap_fsdecoded(result[i])
     return space.newlist(result_w)
@@ -921,7 +921,7 @@
 @unwrap_spec(fd=c_int)
 def get_inheritable(space, fd):
     try:
-        return space.wrap(rposix.get_inheritable(fd))
+        return space.newbool(rposix.get_inheritable(fd))
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
 
@@ -947,7 +947,7 @@
         rposix.c_close(fd2)
         rposix.c_close(fd1)
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+    return space.newtuple([space.newint(fd1), space.newint(fd2)])
 
 @unwrap_spec(flags=c_int)
 def pipe2(space, flags):
@@ -955,7 +955,7 @@
         fd1, fd2 = rposix.pipe2(flags)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+    return space.newtuple([space.newint(fd1), space.newint(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
              follow_symlinks=bool)
@@ -1136,7 +1136,7 @@
 def umask(space, mask):
     "Set the current numeric umask and return the previous umask."
     prevmask = os.umask(mask)
-    return space.wrap(prevmask)
+    return space.newint(prevmask)
 
 def getpid(space):
     "Return the current process id."
@@ -1144,7 +1144,7 @@
         pid = os.getpid()
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(pid)
+    return space.newint(pid)
 
 @unwrap_spec(pid=c_int, signal=c_int)
 def kill(space, pid, signal):
@@ -1305,7 +1305,7 @@
 
 def fork(space):
     pid, irrelevant = _run_forking_function(space, "F")
-    return space.wrap(pid)
+    return space.newint(pid)
 
 def openpty(space):
     "Open a pseudo-terminal, returning open fd's for both master and slave end."
@@ -1320,12 +1320,12 @@
         if slave_fd >= 0:
             rposix.c_close(slave_fd)
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)])
+    return space.newtuple([space.newint(master_fd), space.newint(slave_fd)])
 
 def forkpty(space):
     pid, master_fd = _run_forking_function(space, "P")
-    return space.newtuple([space.wrap(pid),
-                           space.wrap(master_fd)])
+    return space.newtuple([space.newint(pid),
+                           space.newint(master_fd)])
 
 @unwrap_spec(pid=c_int, options=c_int)
 def waitpid(space, pid, options):
@@ -1339,7 +1339,7 @@
             break
         except OSError as e:
             wrap_oserror(space, e, eintr_retry=True)
-    return space.newtuple([space.wrap(pid), space.wrap(status)])
+    return space.newtuple([space.newint(pid), space.newint(status)])
 
 # missing: waitid()
 
@@ -1429,7 +1429,7 @@
         ret = os.spawnv(mode, path, args)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(ret)
+    return space.newint(ret)
 
 @unwrap_spec(mode=int, path='fsencode')
 def spawnve(space, mode, path, w_argv, w_env):
@@ -1439,7 +1439,7 @@
         ret = os.spawnve(mode, path, args, env)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(ret)
+    return space.newint(ret)
 
 
 @unwrap_spec(
@@ -1590,7 +1590,7 @@
         return time, 0
 
 def convert_ns(space, w_ns_time):
-    w_billion = space.wrap(1000000000)
+    w_billion = space.newint(1000000000)
     w_res = space.divmod(w_ns_time, w_billion)
     res_w = space.fixedview(w_res)
     time_int = space.int_w(res_w[0])
@@ -1611,7 +1611,7 @@
            for i in [r[0], r[1], r[2], r[3], r[4]]]
     w_tuple = space.newtuple(l_w)
     w_uname_result = space.getattr(space.getbuiltinmodule(os.name),
-                                   space.wrap('uname_result'))
+                                   space.newtext('uname_result'))
     return space.call_function(w_uname_result, w_tuple)
 
 def getuid(space):
@@ -1740,7 +1740,7 @@
 
     Return the current process group id.
     """
-    return space.wrap(os.getpgrp())
+    return space.newint(os.getpgrp())
 
 def setpgrp(space):
     """ setpgrp()
@@ -1758,7 +1758,7 @@
 
     Return the parent's process id.
     """
-    return space.wrap(os.getppid())
+    return space.newint(os.getppid())
 
 @unwrap_spec(pid=c_int)
 def getpgid(space, pid):
@@ -1770,7 +1770,7 @@
         pgid = os.getpgid(pid)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(pgid)
+    return space.newint(pgid)
 
 @unwrap_spec(pid=c_int, pgrp=c_int)
 def setpgid(space, pid, pgrp):
@@ -1816,7 +1816,7 @@
         sid = os.getsid(pid)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(sid)
+    return space.newint(sid)
 
 def setsid(space):
     """ setsid()
@@ -1839,7 +1839,7 @@
         pgid = os.tcgetpgrp(fd)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(pgid)
+    return space.newint(pgid)
 
 @unwrap_spec(fd=c_int, pgid=c_gid_t)
 def tcsetpgrp(space, fd, pgid):
@@ -1910,7 +1910,7 @@
         returned_priority = rposix.getpriority(which, who)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(returned_priority)
+    return space.newint(returned_priority)
 
 @unwrap_spec(which=int, who=int, priority=int)
 def setpriority(space, which, who, priority):
@@ -1927,7 +1927,7 @@
     if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
-            return space.wrap(getattr(os, name)(status))
+            return space.newint(getattr(os, name)(status))
     else:
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
@@ -1967,7 +1967,7 @@
         res = os.sysconf(num)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(res)
+    return space.newint(res)
 
 @unwrap_spec(fd=c_int)
 def fpathconf(space, fd, w_name):
@@ -1976,7 +1976,7 @@
         res = os.fpathconf(fd, num)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(res)
+    return space.newint(res)
 
 @unwrap_spec(path=path_or_fd(allow_fd=hasattr(os, 'fpathconf')))
 def pathconf(space, path, w_name):
@@ -1991,7 +1991,7 @@
             res = os.pathconf(path.as_bytes, num)
         except OSError as e:
             raise wrap_oserror2(space, e, path.w_path, eintr_retry=False)
-    return space.wrap(res)
+    return space.newint(res)
 
 def confstr(space, w_name):
     num = confname_w(space, w_name, os.confstr_names)
@@ -1999,7 +1999,7 @@
         res = os.confstr(num)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(res)
+    return space.newtext(res)
 
 @unwrap_spec(
     uid=c_uid_t, gid=c_gid_t,
@@ -2098,24 +2098,24 @@
         load = os.getloadavg()
     except OSError:
         raise oefmt(space.w_OSError, "Load averages are unobtainable")
-    return space.newtuple([space.wrap(load[0]),
-                           space.wrap(load[1]),
-                           space.wrap(load[2])])
+    return space.newtuple([space.newfloat(load[0]),
+                           space.newfloat(load[1]),
+                           space.newfloat(load[2])])
 
 @unwrap_spec(major=c_int, minor=c_int)
 def makedev(space, major, minor):
     result = os.makedev(major, minor)
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(device="c_uint")
 def major(space, device):
     result = os.major(intmask(device))
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(device="c_uint")
 def minor(space, device):
     result = os.minor(intmask(device))
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(increment=c_int)
 def nice(space, increment):
@@ -2125,7 +2125,7 @@
         res = os.nice(increment)
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap(res)
+    return space.newint(res)
 
 class SigCheck:
     pass
@@ -2169,14 +2169,14 @@
         return space.w_None
     if _WIN32:
         if fd == 0:
-            return space.wrap('cp%d' % rwin32.GetConsoleCP())
+            return space.newtext('cp%d' % rwin32.GetConsoleCP())
         if fd in (1, 2):
-            return space.wrap('cp%d' % rwin32.GetConsoleOutputCP())
+            return space.newtext('cp%d' % rwin32.GetConsoleOutputCP())
     from rpython.rlib import rlocale
     if rlocale.HAVE_LANGINFO:
         codeset = rlocale.nl_langinfo(rlocale.CODESET)
         if codeset:
-            return space.wrap(codeset)
+            return space.newtext(codeset)
     return space.w_None
 
 if _WIN32:
@@ -2188,9 +2188,9 @@
             info = nt._getfileinformation(fd)
         except OSError as e:
             raise wrap_oserror(space, e, eintr_retry=False)
-        return space.newtuple([space.wrap(info[0]),
-                               space.wrap(info[1]),
-                               space.wrap(info[2])])
+        return space.newtuple([space.newint(info[0]),
+                               space.newint(info[1]),
+                               space.newint(info[2])])
 
     def _getfinalpathname(space, w_path):
         path = space.unicode_w(w_path)
@@ -2198,10 +2198,10 @@
             result = nt._getfinalpathname(path)
         except nt.LLNotImplemented as e:
             raise OperationError(space.w_NotImplementedError,
-                                 space.wrap(e.msg))
+                                 space.newtext(e.msg))
         except OSError as e:
             raise wrap_oserror2(space, e, w_path, eintr_retry=False)
-        return space.wrap(result)
+        return space.newunicode(result)
 
 
 def chflags():
@@ -2304,20 +2304,20 @@
             success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
             if not success:
                 raise rwin32.lastSavedWindowsError()
-            w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
-            w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
+            w_columns = space.newint(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
+            w_lines = space.newint(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
     else:
         with lltype.scoped_alloc(rposix.WINSIZE) as winsize:
             failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
             if failed:
                 raise exception_from_saved_errno(space, space.w_OSError)
 
-            w_columns = space.wrap(r_uint(winsize.c_ws_col))
-            w_lines = space.wrap(r_uint(winsize.c_ws_row))
+            w_columns = space.newint(r_uint(winsize.c_ws_col))
+            w_lines = space.newint(r_uint(winsize.c_ws_row))
 
     w_tuple = space.newtuple([w_columns, w_lines])
     w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
-                                    space.wrap('terminal_size'))
+                                    space.newtext('terminal_size'))
 
     return space.call_function(w_terminal_size, w_tuple)
 
@@ -2325,7 +2325,7 @@
     count = rposix.cpu_count()
     if count <= 0:
         return space.w_None
-    return space.wrap(count)
+    return space.newint(count)
 
 @unwrap_spec(fd=c_int)
 def get_blocking(space, fd):
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -16,15 +16,15 @@
         w_codes = space.newdict()
         w_messages = space.newdict()
         for name in interp_pyexpat.xml_error_list:
-            w_name = space.wrap(name)
+            w_name = space.newtext(name)
             num = getattr(interp_pyexpat, name)
-            w_num = space.wrap(num)
+            w_num = space.newint(num)
             w_message = interp_pyexpat.ErrorString(space, num)
             space.setattr(self, w_name, w_message)
             space.setitem(w_codes, w_message, w_num)
             space.setitem(w_messages, w_num, w_message)
-        space.setattr(self, space.wrap("codes"), w_codes)
-        space.setattr(self, space.wrap("messages"), w_messages)
+        space.setattr(self, space.newtext("codes"), w_codes)
+        space.setattr(self, space.newtext("messages"), w_messages)
 
 class ModelModule(MixedModule):
     "Definition of pyexpat.model module."
@@ -36,7 +36,7 @@
         space = self.space
         for name in interp_pyexpat.xml_model_list:
             value = getattr(interp_pyexpat, name)
-            space.setattr(self, space.wrap(name), space.wrap(value))
+            space.setattr(self, space.newtext(name), space.newtext(value))
 
 class Module(MixedModule):
     "Python wrapper for Expat parser."
@@ -52,7 +52,7 @@
         'ExpatError':    'space.fromcache(interp_pyexpat.Cache).w_error',
         'error':         'space.fromcache(interp_pyexpat.Cache).w_error',
 
-        '__version__':   'space.wrap("85819")',
+        '__version__':   'space.newtext("85819")',
         }
 
     submodules = {
@@ -63,11 +63,11 @@
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
                  'XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE',
                  'XML_PARAM_ENTITY_PARSING_ALWAYS']:
-        interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,)
+        interpleveldefs[name] = 'space.newint(interp_pyexpat.%s)' % (name,)
 
     def startup(self, space):
         from pypy.module.pyexpat import interp_pyexpat
         w_ver = interp_pyexpat.get_expat_version(space)
-        space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver)
+        space.setattr(self, space.newtext("EXPAT_VERSION"), w_ver)
         w_ver = interp_pyexpat.get_expat_version_info(space)
-        space.setattr(self, space.wrap("version_info"), w_ver)
+        space.setattr(self, space.newtext("version_info"), w_ver)
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -488,7 +488,7 @@
 
     def w_convert(self, space, s):
         from pypy.interpreter.unicodehelper import decode_utf8
-        return space.wrap(decode_utf8(space, s))
+        return space.newtext(s)
 
     def w_convert_charp(self, space, data):
         if data:
@@ -591,7 +591,7 @@
         translationmap = space.unicode_w(
             space.call_method(
                 space.newbytes(self.all_chars), "decode",
-                space.wrap(name), space.wrap("replace")))
+                space.newtext(name), space.newtext("replace")))
 
         if len(translationmap) != 256:
             raise oefmt(space.w_ValueError,
@@ -626,7 +626,7 @@
 
 
     def get_namespace_prefixes(self, space):
-        return space.wrap(self.ns_prefixes)
+        return space.newbool(self.ns_prefixes)
 
     def set_namespace_prefixes(self, space, w_value):
         self.ns_prefixes = space.bool_w(w_value)
@@ -655,14 +655,14 @@
             exc = self.set_error(space, XML_GetErrorCode(self.itself))
             raise exc
         self.flush_character_buffer(space)
-        return space.wrap(res)
+        return space.newint(res)
 
     def ParseFile(self, space, w_file):
         """ParseFile(file)
 Parse XML data from file-like object."""
         eof = False
         while not eof:
-            w_data = space.call_method(w_file, 'read', space.wrap(2048))
+            w_data = space.call_method(w_file, 'read', space.newint(2048))
             eof = space.len_w(w_data) == 0
             w_res = self.Parse(space, w_data, isfinal=eof)
         return w_res
@@ -696,13 +696,13 @@
         for i in range(NB_HANDLERS):
             parser.handlers[i] = self.handlers[i]
 
-        return space.wrap(parser)
+        return parser
 
     def flush_character_buffer(self, space):
         if not self.buffer:
             return
         w_data = space.call_function(
-            space.getattr(space.wrap(''), space.wrap('join')),
+            space.getattr(space.newtext(''), space.newtext('join')),
             space.newlist(self.buffer))
         self.buffer = []
         self.buffer_used = 0
@@ -718,28 +718,28 @@
         colno = XML_GetCurrentColumnNumber(self.itself)
         msg = "%s: line %d, column %d" % (err, lineno, colno)
         w_errorcls = space.fromcache(Cache).w_error
-        w_error = space.call_function(w_errorcls, space.wrap(msg))
-        space.setattr(w_error, space.wrap("code"), space.wrap(code))
-        space.setattr(w_error, space.wrap("offset"), space.wrap(colno))
-        space.setattr(w_error, space.wrap("lineno"), space.wrap(lineno))
+        w_error = space.call_function(w_errorcls, space.newtext(msg))
+        space.setattr(w_error, space.newtext("code"), space.newint(code))
+        space.setattr(w_error, space.newtext("offset"), space.newint(colno))
+        space.setattr(w_error, space.newtext("lineno"), space.newint(lineno))
 
         self.w_error = w_error
         return OperationError(w_errorcls, w_error)
 
     def descr_ErrorCode(self, space):
-        return space.wrap(XML_GetErrorCode(self.itself))
+        return space.newint(XML_GetErrorCode(self.itself))
 
     def descr_ErrorLineNumber(self, space):
-        return space.wrap(XML_GetErrorLineNumber(self.itself))
+        return space.newint(XML_GetErrorLineNumber(self.itself))
 
     def descr_ErrorColumnNumber(self, space):
-        return space.wrap(XML_GetErrorColumnNumber(self.itself))
+        return space.newint(XML_GetErrorColumnNumber(self.itself))
 
     def descr_ErrorByteIndex(self, space):
-        return space.wrap(XML_GetErrorByteIndex(self.itself))
+        return space.newint(XML_GetErrorByteIndex(self.itself))
 
     def get_buffer_size(self, space):
-        return space.wrap(self.buffer_size)
+        return space.newint(self.buffer_size)
     def set_buffer_size(self, space, w_value):
         value = space.getindex_w(w_value, space.w_OverflowError)
         if value <= 0:
@@ -749,7 +749,7 @@
         self.buffer_size = value
 
     def get_buffer_text(self, space):
-        return space.wrap(self.buffer is not None)
+        return space.newbool(self.buffer is not None)
     def set_buffer_text(self, space, w_value):
         if space.is_true(w_value):
             self.buffer = []
@@ -767,7 +767,7 @@
 
 def bool_property(name, cls, doc=None):
     def fget(space, obj):
-        return space.wrap(getattr(obj, name))
+        return space.newbool(getattr(obj, name))
     def fset(space, obj, value):
         setattr(obj, name, space.bool_w(value))
     return GetSetProperty(fget, fset, cls=cls, doc=doc)
@@ -863,11 +863,11 @@
     XML_SetUnknownEncodingHandler(
         parser.itself, UnknownEncodingHandlerData_callback,
         rffi.cast(rffi.VOIDP, parser.id))
-    return space.wrap(parser)
+    return parser
 
 @unwrap_spec(code=int)
 def ErrorString(space, code):
     """ErrorString(errno) -> string
 Returns string error for given number."""
-    return space.wrap(rffi.charp2str(XML_ErrorString(code)))
+    return space.newtext(rffi.charp2str(XML_ErrorString(code)))
 
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -273,7 +273,7 @@
                     
                     _setinfo(space, w_info, "GetSystemTimeAsFileTime()",
                              time_increment[0] * 1e-7, False, True)
-            return space.wrap(tv_sec + tv_usec * 1e-6)
+            return space.newfloat(tv_sec + tv_usec * 1e-6)
 else:
     if HAVE_GETTIMEOFDAY:
         if GETTIMEOFDAY_NO_TZ:
@@ -293,7 +293,7 @@
                 if rffi.cast(rffi.LONG, errcode) == 0:
                     if w_info is not None:
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
-                    return space.wrap(
+                    return space.newfloat(
                         widen(timeval.c_tv_sec) +
                         widen(timeval.c_tv_usec) * 1e-6)
         if HAVE_FTIME:
@@ -304,11 +304,11 @@
                 if w_info is not None:
                     _setinfo(space, w_info, "ftime()", 1e-3,
                              False, True) 
-            return space.wrap(result)
+            return space.newfloat(result)
         else:
             if w_info:
                 _setinfo(space, w_info, "time()", 1.0, False, True)
-            return space.wrap(c_time(lltype.nullptr(rffi.TIME_TP.TO)))
+            return space.newint(c_time(lltype.nullptr(rffi.TIME_TP.TO)))
 
 TM_P = lltype.Ptr(tm)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
@@ -437,11 +437,11 @@
                 daylight = int(janzone != julyzone)
                 tzname = [janname, julyname]
 
-    _set_module_object(space, "timezone", space.wrap(timezone))
-    _set_module_object(space, 'daylight', space.wrap(daylight))
-    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
+    _set_module_object(space, "timezone", space.newint(timezone))
+    _set_module_object(space, 'daylight', space.newint(daylight))
+    tzname_w = [space.newtext(tzname[0]), space.newtext(tzname[1])]
     _set_module_object(space, 'tzname', space.newtuple(tzname_w))
-    _set_module_object(space, 'altzone', space.wrap(altzone))
+    _set_module_object(space, 'altzone', space.newint(altzone))
 
 def _get_error_msg():
     errno = rposix.get_saved_errno()
@@ -508,12 +508,12 @@
 
 def _get_module_object(space, obj_name):
     w_module = space.getbuiltinmodule('time')
-    w_obj = space.getattr(w_module, space.wrap(obj_name))
+    w_obj = space.getattr(w_module, space.newtext(obj_name))
     return w_obj
 
 def _set_module_object(space, obj_name, w_obj_value):
     w_module = space.getbuiltinmodule('time')
-    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
+    space.setattr(w_module, space.newtext(obj_name), w_obj_value)
 
 def _get_inttime(space, w_seconds):
     # w_seconds can be a wrapped None (it will be automatically wrapped
@@ -536,22 +536,22 @@
 
 def _tm_to_tuple(space, t):
     time_tuple = [
-        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
-        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
-        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
-        space.wrap(rffi.getintfield(t, 'c_tm_min')),
-        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
-        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
-        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
+        space.newint(rffi.getintfield(t, 'c_tm_year') + 1900),
+        space.newint(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
+        space.newint(rffi.getintfield(t, 'c_tm_mday')),
+        space.newint(rffi.getintfield(t, 'c_tm_hour')),
+        space.newint(rffi.getintfield(t, 'c_tm_min')),
+        space.newint(rffi.getintfield(t, 'c_tm_sec')),
+        space.newint((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
+        space.newint(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
+        space.newint(rffi.getintfield(t, 'c_tm_isdst'))]
 
     if HAS_TM_ZONE:
         # CPython calls PyUnicode_DecodeLocale here should we do the same?
         tm_zone = decode_utf8(space, rffi.charp2str(t.c_tm_zone),
                               allow_surrogates=True)
         extra = [space.newunicode(tm_zone),
-                 space.wrap(rffi.getintfield(t, 'c_tm_gmtoff'))]
+                 space.newint(rffi.getintfield(t, 'c_tm_gmtoff'))]
         w_time_tuple = space.newtuple(time_tuple + extra)
     else:
         w_time_tuple = space.newtuple(time_tuple)
@@ -573,7 +573,7 @@
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
-                                 space.wrap(_get_error_msg()))
+                space.newtext(_get_error_msg()))
         return pbuf
 
     tup_w = space.fixedview(w_tup)
@@ -671,7 +671,7 @@
                             res = 1e-9
                         _setinfo(space, w_info, "clock_gettime(CLOCK_REALTIME)",
                                  res, False, True)
-                return space.wrap(_timespec_to_seconds(timespec))
+                return space.newfloat(_timespec_to_seconds(timespec))
     else:
         return gettimeofday(space, w_info)
 
@@ -710,15 +710,15 @@
 def _asctime(space, t_ref):
     # Inspired by Open Group reference implementation available at
     # http://pubs.opengroup.org/onlinepubs/009695399/functions/asctime.html
-    w, getif = space.wrap, rffi.getintfield
-    args = [w(_wday_names[getif(t_ref, 'c_tm_wday')]),
-            w(_mon_names[getif(t_ref, 'c_tm_mon')]),
-            w(getif(t_ref, 'c_tm_mday')),
-            w(getif(t_ref, 'c_tm_hour')),
-            w(getif(t_ref, 'c_tm_min')),
-            w(getif(t_ref, 'c_tm_sec')),
-            w(getif(t_ref, 'c_tm_year'))]
-    return space.mod(w("%.3s %.3s%3d %.2d:%.2d:%.2d %d"),
+    getif = rffi.getintfield
+    args = [space.newtext(_wday_names[getif(t_ref, 'c_tm_wday')]),
+            space.newtext(_mon_names[getif(t_ref, 'c_tm_mon')]),
+            space.newint(getif(t_ref, 'c_tm_mday')),
+            space.newint(getif(t_ref, 'c_tm_hour')),
+            space.newint(getif(t_ref, 'c_tm_min')),
+            space.newint(getif(t_ref, 'c_tm_sec')),
+            space.newint(getif(t_ref, 'c_tm_year'))]
+    return space.mod(space.newtext("%.3s %.3s%3d %.2d:%.2d:%.2d %d"),
                      space.newtuple(args))
 
 def gmtime(space, w_seconds=None):
@@ -738,7 +738,7 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
+        raise OperationError(space.w_ValueError, space.newtext(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def localtime(space, w_seconds=None):
@@ -755,7 +755,7 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_OSError, space.wrap(_get_error_msg()))
+        raise OperationError(space.w_OSError, space.newtext(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def mktime(space, w_tup):
@@ -772,7 +772,7 @@
     if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
         raise oefmt(space.w_OverflowError, "mktime argument out of range")
 
-    return space.wrap(float(tt))
+    return space.newfloat(float(tt))
 
 if HAS_CLOCK_GETTIME:
     def _timespec_to_seconds(timespec):
@@ -785,7 +785,7 @@
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
             secs = _timespec_to_seconds(timespec)
-        return space.wrap(secs)
+        return space.newfloat(secs)
 
     @unwrap_spec(clk_id='c_int', secs=float)
     def clock_settime(space, clk_id, secs):
@@ -805,7 +805,7 @@
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
             secs = _timespec_to_seconds(timespec)
-        return space.wrap(secs)
+        return space.newfloat(secs)
 
 if _POSIX:
     def tzset(space):
@@ -872,7 +872,7 @@
                 # e.g. an empty format, or %Z when the timezone
                 # is unknown.
                 result = rffi.charp2strn(outbuf, intmask(buflen))
-                return space.wrap(result)
+                return space.newtext(result)
         finally:
             lltype.free(outbuf, flavor='raw')
         i += i
@@ -912,7 +912,7 @@
                             rwin32.lastSavedWindowsError("GetSystemTimeAdjustment"))
                     resolution = resolution * time_increment[0]
                 _setinfo(space, w_info, implementation, resolution, True, False)
-            return space.wrap(result)
+            return space.newfloat(result)
 
     elif _MACOSX:
         c_mach_timebase_info = external('mach_timebase_info',
@@ -935,7 +935,7 @@
                 _setinfo(space, w_info, "mach_absolute_time()", res, True, False)
             secs = nanosecs / 10**9
             rest = nanosecs % 10**9
-            return space.wrap(float(secs) + float(rest) * 1e-9)
+            return space.newfloat(float(secs) + float(rest) * 1e-9)
 
     else:
         assert _POSIX
@@ -985,7 +985,7 @@
             if w_info is not None:
                 _setinfo(space, w_info, "QueryPerformanceCounter()", resolution,
                          True, False)
-            return space.wrap(float(diff) / time_state.divisor)
+            return space.newfloat(float(diff) / time_state.divisor)
 
     def perf_counter(space, w_info=None):
         try:
@@ -1025,7 +1025,7 @@
                           r_ulonglong(user_time.c_dwHighDateTime) << 32)
         if w_info is not None:
             _setinfo(space, w_info, "GetProcessTimes()", 1e-7, True, False)
-        return space.wrap((float(kernel_time2) + float(user_time2)) * 1e-7)
+        return space.newfloat((float(kernel_time2) + float(user_time2)) * 1e-7)
 else:
     have_times = hasattr(rposix, 'c_times')
 
@@ -1051,7 +1051,7 @@
                                 res = 1e-9
                         _setinfo(space, w_info,
                                  implementation, res, True, False)
-                    return space.wrap(_timespec_to_seconds(timespec))
+                    return space.newfloat(_timespec_to_seconds(timespec))
 
         if True: # XXX available except if it isn't?
             from rpython.rlib.rtime import (c_getrusage, RUSAGE, RUSAGE_SELF,
@@ -1062,8 +1062,8 @@
                     if w_info is not None:
                         _setinfo(space, w_info,
                                  "getrusage(RUSAGE_SELF)", 1e-6, True, False)
-                    return space.wrap(decode_timeval(rusage.c_ru_utime) +
-                                      decode_timeval(rusage.c_ru_stime))
+                    return space.newfloat(decode_timeval(rusage.c_ru_utime) +
+                                          decode_timeval(rusage.c_ru_stime))
         if have_times:
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
@@ -1076,7 +1076,7 @@
                         _setinfo(space, w_info, "times()",
                                  1.0 / rposix.CLOCK_TICKS_PER_SECOND,
                                  True, False)
-                    return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
+                    return space.newfloat(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
         return clock(space)
 
 _clock = external('clock', [], rposix.CLOCK_T)
@@ -1099,11 +1099,11 @@
     if w_info is not None:
         _setinfo(space, w_info,
                  "clock()", 1.0 / CLOCKS_PER_SEC, True, False)
-    return space.wrap(float(value) / CLOCKS_PER_SEC)
+    return space.newfloat(float(value) / CLOCKS_PER_SEC)
 
 
 def _setinfo(space, w_info, impl, res, mono, adj):
-    space.setattr(w_info, space.wrap('implementation'), space.wrap(impl))
-    space.setattr(w_info, space.wrap('resolution'), space.wrap(res))
-    space.setattr(w_info, space.wrap('monotonic'), space.wrap(mono))
-    space.setattr(w_info, space.wrap('adjustable'), space.wrap(adj))
+    space.setattr(w_info, space.newtext('implementation'), space.newtext(impl))
+    space.setattr(w_info, space.newtext('resolution'), space.newfloat(res))
+    space.setattr(w_info, space.newtext('monotonic'), space.newbool(mono))
+    space.setattr(w_info, space.newtext('adjustable'), space.newbool(adj))
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -101,25 +101,25 @@
         try:
             code = self._lookup(name.upper())
         except KeyError:
-            msg = space.mod(space.wrap("undefined character name '%s'"), space.wrap(name))
+            msg = space.mod(space.newtext("undefined character name '%s'"), space.newtext(name))
             raise OperationError(space.w_KeyError, msg)
-        return space.wrap(code)
+        return space.newint(code)
 
     @unwrap_spec(name=str)
     def lookup(self, space, name):
         try:
             code = self._lookup(name.upper(), with_named_sequence=True)
         except KeyError:
-            msg = space.mod(space.wrap("undefined character name '%s'"), space.wrap(name))
+            msg = space.mod(space.newtext("undefined character name '%s'"), space.newtext(name))
             raise OperationError(space.w_KeyError, msg)
 
         # The code may be a named sequence
         sequence = self._lookup_named_sequence(code)
         if sequence is not None:
             # named sequences only contain UCS2 codes, no surrogates &co.
-            return space.wrap(sequence)
+            return space.newunicode(sequence)
 
-        return space.wrap(code_to_unichr(code))
+        return space.newunicode(code_to_unichr(code))
 
     def name(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
@@ -129,12 +129,12 @@
             if w_default is not None:
                 return w_default
             raise oefmt(space.w_ValueError, "no such name")
-        return space.wrap(name)
+        return space.newtext(name)
 
     def decimal(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
         try:
-            return space.wrap(self._decimal(code))
+            return space.newint(self._decimal(code))
         except KeyError:
             pass
         if w_default is not None:
@@ -144,7 +144,7 @@
     def digit(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
         try:
-            return space.wrap(self._digit(code))
+            return space.newint(self._digit(code))
         except KeyError:
             pass
         if w_default is not None:
@@ -154,7 +154,7 @@
     def numeric(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
         try:
-            return space.wrap(self._numeric(code))
+            return space.newfloat(self._numeric(code))
         except KeyError:
             pass
         if w_default is not None:
@@ -163,28 +163,28 @@
 
     def category(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._category(code))
+        return space.newtext(self._category(code))
 
     def east_asian_width(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._east_asian_width(code))
+        return space.newtext(self._east_asian_width(code))
 
     def bidirectional(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._bidirectional(code))
+        return space.newtext(self._bidirectional(code))
 
     def combining(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._combining(code))
+        return space.newint(self._combining(code))
 
     def mirrored(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
         # For no reason, unicodedata.mirrored() returns an int, not a bool
-        return space.wrap(int(self._mirrored(code)))
+        return space.newint(int(self._mirrored(code)))
 
     def decomposition(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._decomposition(code))
+        return space.newtext(self._decomposition(code))
 
     @unwrap_spec(form=str)
     def normalize(self, space, form, w_unistr):
@@ -213,7 +213,7 @@
         resultlen = len(result)
         # Expand the character
         for i in range(strlen):
-            ch = space.int_w(space.ord(space.getitem(w_unistr, space.wrap(i))))
+            ch = space.int_w(space.ord(space.getitem(w_unistr, space.newint(i))))
             # Do Hangul decomposition
             if SBase <= ch < SBase + SCount:
                 SIndex = ch - SBase
@@ -268,10 +268,10 @@
                 result[0] = ch
 
         if not composed: # If decomposed normalization we are done
-            return space.wrap(u''.join([unichr(i) for i in result[:j]]))
+            return space.newunicode(u''.join([unichr(i) for i in result[:j]]))
 
         if j <= 1:
-            return space.wrap(u''.join([unichr(i) for i in result[:j]]))
+            return space.newunicode(u''.join([unichr(i) for i in result[:j]]))
 
         current = result[0]
         starter_pos = 0
@@ -318,7 +318,7 @@
 
         result[starter_pos] = current
 
-        return space.wrap(u''.join([unichr(i) for i in result[:next_insert]]))
+        return space.newunicode(u''.join([unichr(i) for i in result[:next_insert]]))
 
 
 methods = {}
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -266,7 +266,7 @@
 
     @specialize.argtype(1)
     def newint(self, intval):
-        if self.config.objspace.std.withsmalllong:
+        if self.config.objspace.std.withsmalllong and isinstance(intval, base_int):
             from pypy.objspace.std.smalllongobject import W_SmallLongObject
             from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
             from rpython.rlib.rarithmetic import longlongmax

From pypy.commits at gmail.com  Tue Feb 14 10:55:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 07:55:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 35acc5809da1
Message-ID: <58a32871.496f1c0a.e9a0c.4724@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90112:f9ea8c5f2f1e
Date: 2017-02-14 16:54 +0100
http://bitbucket.org/pypy/pypy/changeset/f9ea8c5f2f1e/

Log:	hg merge 35acc5809da1

diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -13,11 +13,11 @@
 import sys
 
 def make_error(space, msg):
-    return OperationError(space.gettypeobject(W_Error.typedef), space.wrap(msg))
+    return OperationError(space.gettypeobject(W_Error.typedef), space.newtext(msg))
 
 def rewrap_error(space, e):
     return OperationError(space.gettypeobject(W_Error.typedef),
-                          space.wrap(e.message))
+                          space.newtext(e.message))
 
 @unwrap_spec(category=int)
 def setlocale(space, category, w_locale=None):
@@ -31,19 +31,18 @@
         result = rlocale.setlocale(category, locale)
     except rlocale.LocaleError as e:
         raise rewrap_error(space, e)
-    return space.wrap(result)
+    return space.newtext(result)
 
 def _w_copy_grouping(space, text):
-    groups = [ space.wrap(ord(group)) for group in text ]
+    groups = [ space.newint(ord(group)) for group in text ]
     if groups:
-        groups.append(space.wrap(0))
+        groups.append(space.newint(0))
     return space.newlist(groups)
 
 def charp2uni(space, s):
     "Convert a char* pointer to unicode according to the current locale"
-    w_bytes = space.newbytes(rffi.charp2str(s))
     # XXX use mbstowcs()
-    return space.call_method(w_bytes, "decode", space.wrap("utf-8"))
+    return space.newtext(rffi.charp2str(s))
 
 def localeconv(space):
     "() -> dict. Returns numeric and monetary locale-specific parameters."
@@ -51,43 +50,42 @@
 
     # Numeric information
     w_result = space.newdict()
-    w = space.wrap
-    space.setitem(w_result, w("decimal_point"),
+    space.setitem(w_result, space.newtext("decimal_point"),
                   charp2uni(space, lp.c_decimal_point))
-    space.setitem(w_result, w("thousands_sep"),
+    space.setitem(w_result, space.newtext("thousands_sep"),
                   charp2uni(space, lp.c_thousands_sep))
-    space.setitem(w_result, w("grouping"),
+    space.setitem(w_result, space.newtext("grouping"),
                   _w_copy_grouping(space, rffi.charp2str(lp.c_grouping)))
-    space.setitem(w_result, w("int_curr_symbol"),
+    space.setitem(w_result, space.newtext("int_curr_symbol"),
                   charp2uni(space, lp.c_int_curr_symbol))
-    space.setitem(w_result, w("currency_symbol"),
+    space.setitem(w_result, space.newtext("currency_symbol"),
                   charp2uni(space, lp.c_currency_symbol))
-    space.setitem(w_result, w("mon_decimal_point"),
+    space.setitem(w_result, space.newtext("mon_decimal_point"),
                   charp2uni(space, lp.c_mon_decimal_point))
-    space.setitem(w_result, w("mon_thousands_sep"),
+    space.setitem(w_result, space.newtext("mon_thousands_sep"),
                   charp2uni(space, lp.c_mon_thousands_sep))
-    space.setitem(w_result, w("mon_grouping"),
+    space.setitem(w_result, space.newtext("mon_grouping"),
                   _w_copy_grouping(space, rffi.charp2str(lp.c_mon_grouping)))
-    space.setitem(w_result, w("positive_sign"),
+    space.setitem(w_result, space.newtext("positive_sign"),
                   charp2uni(space, lp.c_positive_sign))
-    space.setitem(w_result, w("negative_sign"),
+    space.setitem(w_result, space.newtext("negative_sign"),
                   charp2uni(space, lp.c_negative_sign))
-    space.setitem(w_result, w("int_frac_digits"),
-                  w(lp.c_int_frac_digits))
-    space.setitem(w_result, w("frac_digits"),
-                  w(lp.c_frac_digits))
-    space.setitem(w_result, w("p_cs_precedes"),
-                  w(lp.c_p_cs_precedes))
-    space.setitem(w_result, w("p_sep_by_space"),
-                  w(lp.c_p_sep_by_space))
-    space.setitem(w_result, w("n_cs_precedes"),
-                  w(lp.c_n_cs_precedes))
-    space.setitem(w_result, w("n_sep_by_space"),
-                  w(lp.c_n_sep_by_space))
-    space.setitem(w_result, w("p_sign_posn"),
-                  w(lp.c_p_sign_posn))
-    space.setitem(w_result, w("n_sign_posn"),
-                  w(lp.c_n_sign_posn))
+    space.setitem(w_result, space.newtext("int_frac_digits"),
+                  space.newint(lp.c_int_frac_digits))
+    space.setitem(w_result, space.newtext("frac_digits"),
+                  space.newint(lp.c_frac_digits))
+    space.setitem(w_result, space.newtext("p_cs_precedes"),
+                  space.newint(lp.c_p_cs_precedes))
+    space.setitem(w_result, space.newtext("p_sep_by_space"),
+                  space.newint(lp.c_p_sep_by_space))
+    space.setitem(w_result, space.newtext("n_cs_precedes"),
+                  space.newint(lp.c_n_cs_precedes))
+    space.setitem(w_result, space.newtext("n_sep_by_space"),
+                  space.newint(lp.c_n_sep_by_space))
+    space.setitem(w_result, space.newtext("p_sign_posn"),
+                  space.newint(lp.c_p_sign_posn))
+    space.setitem(w_result, space.newtext("n_sign_posn"),
+                  space.newint(lp.c_n_sign_posn))
 
     return w_result
 
@@ -107,7 +105,7 @@
         rffi.free_wcharp(s1_c)
         rffi.free_wcharp(s2_c)
 
-    return space.wrap(result)
+    return space.newint(result)
 
 _strxfrm = rlocale.external('strxfrm',
                     [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T)
@@ -137,7 +135,7 @@
     val = rffi.charp2str(buf)
     lltype.free(buf, flavor="raw")
 
-    return space.wrap(val)
+    return space.newtext(val)
 
 if rlocale.HAVE_LANGINFO:
 
@@ -147,7 +145,7 @@
         Return the value for the locale information associated with key."""
 
         try:
-            return space.wrap(rlocale.nl_langinfo(key))
+            return space.newtext(rlocale.nl_langinfo(key))
         except ValueError:
             raise oefmt(space.w_ValueError, "unsupported langinfo constant")
 
@@ -163,7 +161,7 @@
         Return translation of msg."""
         msg_c = rffi.str2charp(msg)
         try:
-            return space.wrap(rffi.charp2str(_gettext(msg_c)))
+            return space.newtext(rffi.charp2str(_gettext(msg_c)))
         finally:
             rffi.free_charp(msg_c)
 
@@ -198,7 +196,7 @@
                 rffi.free_charp(domain_c)
                 rffi.free_charp(msg_c)
 
-        return space.wrap(result)
+        return space.newtext(result)
 
     _dcgettext = rlocale.external('dcgettext', [rffi.CCHARP, rffi.CCHARP, rffi.INT],
                                                                 rffi.CCHARP)
@@ -234,7 +232,7 @@
                 rffi.free_charp(domain_c)
                 rffi.free_charp(msg_c)
 
-        return space.wrap(result)
+        return space.newtext(result)
 
 
     _textdomain = rlocale.external('textdomain', [rffi.CCHARP], rffi.CCHARP)
@@ -259,7 +257,7 @@
             finally:
                 rffi.free_charp(domain_c)
 
-        return space.wrap(result)
+        return space.newtext(result)
 
     _bindtextdomain = rlocale.external('bindtextdomain', [rffi.CCHARP, rffi.CCHARP],
                                                                 rffi.CCHARP,
@@ -289,8 +287,8 @@
 
         if not dirname:
             errno = rposix.get_saved_errno()
-            raise OperationError(space.w_OSError, space.wrap(errno))
-        return space.wrap(rffi.charp2str(dirname))
+            raise OperationError(space.w_OSError, space.newint(errno))
+        return space.newtext(rffi.charp2str(dirname))
 
     _bind_textdomain_codeset = rlocale.external('bind_textdomain_codeset',
                                     [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP)
@@ -321,9 +319,9 @@
             if not result:
                 return space.w_None
             else:
-                return space.wrap(rffi.charp2str(result))
+                return space.newtext(rffi.charp2str(result))
 
 if sys.platform == 'win32':
     def getdefaultlocale(space):
         language, encoding = rlocale.getdefaultlocale()
-        return space.newtuple([space.wrap(language), space.wrap(encoding)])
+        return space.newtuple([space.newtext(language), space.newtext(encoding)])
diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -15,7 +15,7 @@
             errors = 'strict'
         self.space = space
         self.errors = errors
-        w_codec = space.getattr(space.wrap(self), space.wrap("codec"))
+        w_codec = space.getattr(self, space.newtext("codec"))
         codec = space.interp_w(MultibyteCodec, w_codec)
         self.codec = codec.codec
         self.name = codec.name
@@ -30,7 +30,7 @@
         self._initialize()
 
     def fget_errors(self, space):
-        return space.wrap(self.errors)
+        return space.newtext(self.errors)
 
     def fset_errors(self, space, w_errors):
         self.errors = space.str_w(w_errors)
@@ -66,14 +66,14 @@
         pos = c_codecs.pypy_cjk_dec_inbuf_consumed(self.decodebuf)
         assert 0 <= pos <= len(object)
         self.pending = object[pos:]
-        return space.wrap(output)
+        return space.newunicode(output)
 
 
 @unwrap_spec(errors="str_or_None")
 def mbidecoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalDecoder, w_subtype)
     r.__init__(space, errors)
-    return space.wrap(r)
+    return r
 
 MultibyteIncrementalDecoder.typedef = TypeDef(
     'MultibyteIncrementalDecoder',
@@ -122,7 +122,7 @@
 def mbiencoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalEncoder, w_subtype)
     r.__init__(space, errors)
-    return space.wrap(r)
+    return r
 
 MultibyteIncrementalEncoder.typedef = TypeDef(
     'MultibyteIncrementalEncoder',
diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py
--- a/pypy/module/_random/interp_random.py
+++ b/pypy/module/_random/interp_random.py
@@ -13,7 +13,7 @@
     x = space.allocate_instance(W_Random, w_subtype)
     x = space.interp_w(W_Random, x)
     W_Random.__init__(x, space, w_anything)
-    return space.wrap(x)
+    return x
 
 
 class W_Random(W_Root):
@@ -32,7 +32,7 @@
                 w_n = space.abs(w_n)
             else:
                 n = space.hash_w(w_n)
-                w_n = space.wrap(r_uint(n))
+                w_n = space.newint(r_uint(n))
         key = []
         w_one = space.newint(1)
         w_two = space.newint(2)
@@ -52,7 +52,7 @@
     def getstate(self, space):
         state = [None] * (rrandom.N + 1)
         for i in range(rrandom.N):
-            state[i] = space.wrap(widen(self._rnd.state[i]))
+            state[i] = space.newint(widen(self._rnd.state[i]))
         state[rrandom.N] = space.newlong(self._rnd.index)
         return space.newtuple(state)
 
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -19,11 +19,11 @@
 
 @unwrap_spec(char_ord=int, flags=int)
 def w_getlower(space, char_ord, flags):
-    return space.wrap(getlower(char_ord, flags))
+    return space.newint(getlower(char_ord, flags))
 
 
 def w_getcodesize(space):
-    return space.wrap(CODESIZE)
+    return space.newint(CODESIZE)
 
 # use the same version of unicodedb as the standard objspace
 import pypy.objspace.std.unicodeobject
@@ -73,20 +73,20 @@
 
 
 def import_re(space):
-    w_import = space.getattr(space.builtin, space.wrap("__import__"))
-    return space.call_function(w_import, space.wrap("re"))
+    w_import = space.getattr(space.builtin, space.newtext("__import__"))
+    return space.call_function(w_import, space.newtext("re"))
 
 def matchcontext(space, ctx):
     try:
         return rsre_core.match_context(ctx)
     except rsre_core.Error as e:
-        raise OperationError(space.w_RuntimeError, space.wrap(e.msg))
+        raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
 def searchcontext(space, ctx):
     try:
         return rsre_core.search_context(ctx)
     except rsre_core.Error as e:
-        raise OperationError(space.w_RuntimeError, space.wrap(e.msg))
+        raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
 # ____________________________________________________________
 #
@@ -128,7 +128,7 @@
         else:
             usep = u', '
             uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
-        return space.wrap(u're.compile(%s%s%s)' % (u, usep, uflags))
+        return space.newunicode(u're.compile(%s%s%s)' % (u, usep, uflags))
 
     def fget_groupindex(self, space):
         w_groupindex = self.w_groupindex
@@ -234,7 +234,7 @@
             if not searchcontext(space, ctx):
                 break
             num_groups = self.num_groups
-            w_emptystr = space.wrap("")
+            w_emptystr = space.newtext("")
             if num_groups == 0:
                 w_item = slice_w(space, ctx, ctx.match_start, ctx.match_end,
                                  w_emptystr)
@@ -257,7 +257,7 @@
         # scanner() method.
         ctx = self.make_ctx(w_string, pos, endpos)
         scanner = W_SRE_Scanner(self, ctx)
-        return self.space.wrap(scanner)
+        return scanner
 
     @unwrap_spec(maxsplit=int)
     def split_w(self, w_string, maxsplit=0):
@@ -267,7 +267,7 @@
                 raise oefmt(space.w_ValueError,
                             "split() requires a non-empty pattern match.")
             space.warn(
-                space.wrap("split() requires a non-empty pattern match."),
+                space.newtext("split() requires a non-empty pattern match."),
                 space.w_FutureWarning)
         #
         splitlist = []
@@ -305,7 +305,7 @@
     def subn_w(self, w_repl, w_string, count=0):
         w_item, n = self.subx(w_repl, w_string, count)
         space = self.space
-        return space.newtuple([w_item, space.wrap(n)])
+        return space.newtuple([w_item, space.newint(n)])
 
     def subx(self, w_ptemplate, w_string, count):
         space = self.space
@@ -341,7 +341,7 @@
                     w_ptemplate = space.newbytes(filter_as_string)
                 w_re = import_re(space)
                 w_filter = space.call_method(w_re, '_subx',
-                                             space.wrap(self), w_ptemplate)
+                                             self, w_ptemplate)
                 filter_is_callable = space.is_true(space.callable(w_filter))
         #
         # XXX this is a bit of a mess, but it improves performance a lot
@@ -459,7 +459,7 @@
 def SRE_Pattern__new__(space, w_subtype, w_pattern, flags, w_code,
               groups=0, w_groupindex=None, w_indexgroup=None):
     n = space.len_w(w_code)
-    code = [intmask(space.uint_w(space.getitem(w_code, space.wrap(i))))
+    code = [intmask(space.uint_w(space.getitem(w_code, space.newint(i))))
             for i in range(n)]
     #
     w_srepat = space.allocate_instance(W_SRE_Pattern, w_subtype)
@@ -521,7 +521,7 @@
         u = space.unicode_w(space.repr(w_s))
         if len(u) > 50:
             u = u[:50]
-        return space.wrap(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' % 
+        return space.newunicode(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' % 
                           (start, end, u))
 
     def cannot_copy_w(self):
@@ -573,22 +573,22 @@
     def expand_w(self, w_template):
         space = self.space
         w_re = import_re(space)
-        return space.call_method(w_re, '_expand', space.wrap(self.srepat),
-                                 space.wrap(self), w_template)
+        return space.call_method(w_re, '_expand', self.srepat,
+                                 self, w_template)
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def start_w(self, w_groupnum):
-        return self.space.wrap(self.do_span(w_groupnum)[0])
+        return self.space.newint(self.do_span(w_groupnum)[0])
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def end_w(self, w_groupnum):
-        return self.space.wrap(self.do_span(w_groupnum)[1])
+        return self.space.newint(self.do_span(w_groupnum)[1])
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def span_w(self, w_groupnum):
         start, end = self.do_span(w_groupnum)
-        return self.space.newtuple([self.space.wrap(start),
-                                    self.space.wrap(end)])
+        return self.space.newtuple([self.space.newint(start),
+                                    self.space.newint(end)])
 
     def flatten_marks(self):
         if self.flatten_cache is None:
@@ -632,7 +632,7 @@
         if lastindex < 0:
             return space.w_None
         w_result = space.finditem(self.srepat.w_indexgroup,
-                                  space.wrap(lastindex))
+                                  space.newint(lastindex))
         if w_result is None:
             return space.w_None
         return w_result
@@ -640,14 +640,14 @@
     def fget_lastindex(self, space):
         lastindex = self._last_index()
         if lastindex >= 0:
-            return space.wrap(lastindex)
+            return space.newint(lastindex)
         return space.w_None
 
     def fget_pos(self, space):
-        return space.wrap(self.ctx.original_pos)
+        return space.newint(self.ctx.original_pos)
 
     def fget_endpos(self, space):
-        return space.wrap(self.ctx.end)
+        return space.newint(self.ctx.end)
 
     def fget_regs(self, space):
         space = self.space
@@ -655,11 +655,11 @@
         num_groups = self.srepat.num_groups
         result_w = [None] * (num_groups + 1)
         ctx = self.ctx
-        result_w[0] = space.newtuple([space.wrap(ctx.match_start),
-                                      space.wrap(ctx.match_end)])
+        result_w[0] = space.newtuple([space.newint(ctx.match_start),
+                                      space.newint(ctx.match_end)])
         for i in range(num_groups):
-            result_w[i + 1] = space.newtuple([space.wrap(fmarks[i*2]),
-                                              space.wrap(fmarks[i*2+1])])
+            result_w[i + 1] = space.newtuple([space.newint(fmarks[i*2]),
+                                              space.newint(fmarks[i*2+1])])
         return space.newtuple(result_w)
 
     def fget_string(self, space):
@@ -712,7 +712,7 @@
         # or matching succeeded so far.
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.ctx.match_start > self.ctx.end:
@@ -738,7 +738,7 @@
             nextstart += (ctx.match_start == nextstart)
             self.ctx = ctx.fresh_copy(nextstart)
             match = W_SRE_Match(self.srepat, ctx)
-            return self.space.wrap(match)
+            return match
         else:
             self.ctx.match_start += 1     # obscure corner case
             return None
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -972,7 +972,7 @@
                 else:
                     message = str(e)
                 state.set_exception(OperationError(space.w_SystemError,
-                                                   space.wrap(message)))
+                                                   space.newtext(message)))
             else:
                 failed = False
 
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -41,25 +41,25 @@
     datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw',
                                 track_allocation=False)
 
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
+    w_datetime = PyImport_Import(space, space.newtext("datetime"))
 
-    w_type = space.getattr(w_datetime, space.wrap("date"))
+    w_type = space.getattr(w_datetime, space.newtext("date"))
     datetimeAPI.c_DateType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("datetime"))
+    w_type = space.getattr(w_datetime, space.newtext("datetime"))
     datetimeAPI.c_DateTimeType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("time"))
+    w_type = space.getattr(w_datetime, space.newtext("time"))
     datetimeAPI.c_TimeType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("timedelta"))
+    w_type = space.getattr(w_datetime, space.newtext("timedelta"))
     datetimeAPI.c_DeltaType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("tzinfo"))
+    w_type = space.getattr(w_datetime, space.newtext("tzinfo"))
     datetimeAPI.c_TZInfoType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
@@ -141,7 +141,7 @@
     day = rffi.cast(lltype.Signed, day)
     return space.call_function(
         w_type,
-        space.wrap(year), space.wrap(month), space.wrap(day))
+        space.newint(year), space.newint(month), space.newint(day))
 
 @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
               PyObject, PyTypeObjectPtr], PyObject)
@@ -154,8 +154,8 @@
     usecond = rffi.cast(lltype.Signed, usecond)
     return space.call_function(
         w_type,
-        space.wrap(hour), space.wrap(minute), space.wrap(second),
-        space.wrap(usecond), w_tzinfo)
+        space.newint(hour), space.newint(minute), space.newint(second),
+        space.newint(usecond), w_tzinfo)
 
 @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real,
               rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
@@ -175,18 +175,18 @@
     usecond = rffi.cast(lltype.Signed, usecond)
     return space.call_function(
         w_type,
-        space.wrap(year), space.wrap(month), space.wrap(day),
-        space.wrap(hour), space.wrap(minute), space.wrap(second),
-        space.wrap(usecond), w_tzinfo)
+        space.newint(year), space.newint(month), space.newint(day),
+        space.newint(hour), space.newint(minute), space.newint(second),
+        space.newint(usecond), w_tzinfo)
 
 @cpython_api([PyObject], PyObject)
 def PyDateTime_FromTimestamp(space, w_args):
     """Create and return a new datetime.datetime object given an argument tuple
     suitable for passing to datetime.datetime.fromtimestamp().
     """
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    w_type = space.getattr(w_datetime, space.wrap("datetime"))
-    w_method = space.getattr(w_type, space.wrap("fromtimestamp"))
+    w_datetime = PyImport_Import(space, space.newtext("datetime"))
+    w_type = space.getattr(w_datetime, space.newtext("datetime"))
+    w_method = space.getattr(w_type, space.newtext("fromtimestamp"))
     return space.call(w_method, w_args)
 
 @cpython_api([PyObject], PyObject)
@@ -194,9 +194,9 @@
     """Create and return a new datetime.date object given an argument tuple
     suitable for passing to datetime.date.fromtimestamp().
     """
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    w_type = space.getattr(w_datetime, space.wrap("date"))
-    w_method = space.getattr(w_type, space.wrap("fromtimestamp"))
+    w_datetime = PyImport_Import(space, space.newtext("datetime"))
+    w_type = space.getattr(w_datetime, space.newtext("date"))
+    w_method = space.getattr(w_type, space.newtext("fromtimestamp"))
     return space.call(w_method, w_args)
 
 @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
@@ -213,7 +213,7 @@
     useconds = rffi.cast(lltype.Signed, useconds)
     return space.call_function(
         w_type,
-        space.wrap(days), space.wrap(seconds), space.wrap(useconds))
+        space.newint(days), space.newint(seconds), space.newint(useconds))
 
 # Accessors
 
@@ -221,67 +221,67 @@
 def PyDateTime_GET_YEAR(space, w_obj):
     """Return the year, as a positive int.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("year")))
+    return space.int_w(space.getattr(w_obj, space.newtext("year")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_GET_MONTH(space, w_obj):
     """Return the month, as an int from 1 through 12.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("month")))
+    return space.int_w(space.getattr(w_obj, space.newtext("month")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_GET_DAY(space, w_obj):
     """Return the day, as an int from 1 through 31.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("day")))
+    return space.int_w(space.getattr(w_obj, space.newtext("day")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_HOUR(space, w_obj):
     """Return the hour, as an int from 0 through 23.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("hour")))
+    return space.int_w(space.getattr(w_obj, space.newtext("hour")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_MINUTE(space, w_obj):
     """Return the minute, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("minute")))
+    return space.int_w(space.getattr(w_obj, space.newtext("minute")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_SECOND(space, w_obj):
     """Return the second, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("second")))
+    return space.int_w(space.getattr(w_obj, space.newtext("second")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_MICROSECOND(space, w_obj):
     """Return the microsecond, as an int from 0 through 999999.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("microsecond")))
+    return space.int_w(space.getattr(w_obj, space.newtext("microsecond")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_HOUR(space, w_obj):
     """Return the hour, as an int from 0 through 23.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("hour")))
+    return space.int_w(space.getattr(w_obj, space.newtext("hour")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_MINUTE(space, w_obj):
     """Return the minute, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("minute")))
+    return space.int_w(space.getattr(w_obj, space.newtext("minute")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_SECOND(space, w_obj):
     """Return the second, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("second")))
+    return space.int_w(space.getattr(w_obj, space.newtext("second")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_MICROSECOND(space, w_obj):
     """Return the microsecond, as an int from 0 through 999999.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("microsecond")))
+    return space.int_w(space.getattr(w_obj, space.newtext("microsecond")))
 
 # XXX these functions are not present in the Python API
 # But it does not seem possible to expose a different structure
@@ -289,12 +289,12 @@
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_DAYS(space, w_obj):
-    return space.int_w(space.getattr(w_obj, space.wrap("days")))
+    return space.int_w(space.getattr(w_obj, space.newtext("days")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_SECONDS(space, w_obj):
-    return space.int_w(space.getattr(w_obj, space.wrap("seconds")))
+    return space.int_w(space.getattr(w_obj, space.newtext("seconds")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_MICROSECONDS(space, w_obj):
-    return space.int_w(space.getattr(w_obj, space.wrap("microseconds")))
+    return space.int_w(space.getattr(w_obj, space.newtext("microseconds")))
diff --git a/pypy/module/cpyext/codecs.py b/pypy/module/cpyext/codecs.py
--- a/pypy/module/cpyext/codecs.py
+++ b/pypy/module/cpyext/codecs.py
@@ -6,7 +6,7 @@
 def PyCodec_IncrementalEncoder(space, encoding, errors):
     w_codec = interp_codecs.lookup_codec(space, rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
         return space.call_method(w_codec, "incrementalencoder", w_errors)
     else:
         return space.call_method(w_codec, "incrementalencoder")
@@ -15,7 +15,7 @@
 def PyCodec_IncrementalDecoder(space, encoding, errors):
     w_codec = interp_codecs.lookup_codec(space, rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
         return space.call_method(w_codec, "incrementaldecoder", w_errors)
     else:
         return space.call_method(w_codec, "incrementaldecoder")
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -129,7 +129,7 @@
         key = rffi.charp2str(key_ptr)
         # our dicts dont have a standardized interface, so we need
         # to go through the space
-        space.delitem(w_dict, space.wrap(key))
+        space.delitem(w_dict, space.newtext(key))
         return 0
     else:
         PyErr_BadInternalCall(space)
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -31,7 +31,7 @@
     caller = space.getexecutioncontext().gettopframe_nohidden()
     if caller is not None:
         w_globals = caller.get_w_globals()
-        w_builtins = space.getitem(w_globals, space.wrap('__builtins__'))
+        w_builtins = space.getitem(w_globals, space.newtext('__builtins__'))
         if not space.isinstance_w(w_builtins, space.w_dict):
             w_builtins = w_builtins.getdict(space)
     else:
diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py
--- a/pypy/module/cpyext/floatobject.py
+++ b/pypy/module/cpyext/floatobject.py
@@ -42,7 +42,7 @@
 
 @cpython_api([lltype.Float], PyObject)
 def PyFloat_FromDouble(space, value):
-    return space.wrap(value)
+    return space.newfloat(value)
 
 @cpython_api([PyObject], lltype.Float, error=-1)
 def PyFloat_AsDouble(space, w_obj):
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -63,9 +63,8 @@
     frame = space.FrameClass(space, code, w_globals, outer_func=None)
     d = frame.getorcreatedebug()
     d.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno')
-    w_obj = space.wrap(frame)
-    track_reference(space, py_obj, w_obj)
-    return w_obj
+    track_reference(space, py_obj, frame)
+    return frame
 
 @cpython_api([PyThreadState, PyCodeObject, PyObject, PyObject], PyFrameObject,
              result_is_ll=True)
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -55,7 +55,7 @@
 def function_attach(space, py_obj, w_obj, w_userdata=None):
     py_func = rffi.cast(PyFunctionObject, py_obj)
     assert isinstance(w_obj, Function)
-    py_func.c_func_name = make_ref(space, space.wrap(w_obj.name))
+    py_func.c_func_name = make_ref(space, space.newtext(w_obj.name))
 
 @slot_function([PyObject], lltype.Void)
 def function_dealloc(space, py_obj):
@@ -67,8 +67,8 @@
 def code_attach(space, py_obj, w_obj, w_userdata=None):
     py_code = rffi.cast(PyCodeObject, py_obj)
     assert isinstance(w_obj, PyCode)
-    py_code.c_co_name = make_ref(space, space.wrap(w_obj.co_name))
-    py_code.c_co_filename = make_ref(space, space.wrap(w_obj.co_filename))
+    py_code.c_co_name = make_ref(space, space.newtext(w_obj.co_name))
+    py_code.c_co_filename = make_ref(space, space.newtext(w_obj.co_filename))
     co_flags = 0
     for name, value in ALL_CODE_FLAGS:
         if w_obj.co_flags & getattr(pycode, name):
@@ -88,8 +88,7 @@
 def PyFunction_GetCode(space, w_func):
     """Return the code object associated with the function object op."""
     func = space.interp_w(Function, w_func)
-    w_code = space.wrap(func.code)
-    return w_code      # borrowed ref
+    return func.code      # borrowed ref
 
 @cpython_api([PyObject, PyObject], PyObject)
 def PyMethod_New(space, w_func, w_self):
@@ -113,7 +112,7 @@
     return w_method.w_instance     # borrowed ref
 
 def unwrap_list_of_strings(space, w_list):
-    return [space.str_w(w_item) for w_item in space.fixedview(w_list)]
+    return [space.text_w(w_item) for w_item in space.fixedview(w_list)]
 
 @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
               rffi.INT_real,
@@ -126,42 +125,42 @@
     create a frame, use PyCode_NewEmpty() instead.  Calling
     PyCode_New() directly can bind you to a precise Python
     version since the definition of the bytecode changes often."""
-    return space.wrap(PyCode(space,
-                             argcount=rffi.cast(lltype.Signed, argcount),
-                             kwonlyargcount = 0,  # XXX fix signature
-                             nlocals=rffi.cast(lltype.Signed, nlocals),
-                             stacksize=rffi.cast(lltype.Signed, stacksize),
-                             flags=rffi.cast(lltype.Signed, flags),
-                             code=space.str_w(w_code),
-                             consts=space.fixedview(w_consts),
-                             names=unwrap_list_of_strings(space, w_names),
-                             varnames=unwrap_list_of_strings(space, w_varnames),
-                             filename=space.str0_w(w_filename),
-                             name=space.str_w(w_funcname),
-                             firstlineno=rffi.cast(lltype.Signed, firstlineno),
-                             lnotab=space.str_w(w_lnotab),
-                             freevars=unwrap_list_of_strings(space, w_freevars),
-                             cellvars=unwrap_list_of_strings(space, w_cellvars)))
+    return PyCode(space,
+                  argcount=rffi.cast(lltype.Signed, argcount),
+                  kwonlyargcount = rffi.cast(lltype.Signed, kwonlyargcount),
+                  nlocals=rffi.cast(lltype.Signed, nlocals),
+                  stacksize=rffi.cast(lltype.Signed, stacksize),
+                  flags=rffi.cast(lltype.Signed, flags),
+                  code=space.bytes_w(w_code),
+                  consts=space.fixedview(w_consts),
+                  names=unwrap_list_of_strings(space, w_names),
+                  varnames=unwrap_list_of_strings(space, w_varnames),
+                  filename=space.str0_w(w_filename),
+                  name=space.text_w(w_funcname),
+                  firstlineno=rffi.cast(lltype.Signed, firstlineno),
+                  lnotab=space.bytes_w(w_lnotab),
+                  freevars=unwrap_list_of_strings(space, w_freevars),
+                  cellvars=unwrap_list_of_strings(space, w_cellvars)))
 
 @cpython_api([CONST_STRING, CONST_STRING, rffi.INT_real], PyCodeObject)
 def PyCode_NewEmpty(space, filename, funcname, firstlineno):
     """Creates a new empty code object with the specified source location."""
-    return space.wrap(PyCode(space,
-                             argcount=0,
-                             kwonlyargcount=0,
-                             nlocals=0,
-                             stacksize=0,
-                             flags=0,
-                             code="",
-                             consts=[],
-                             names=[],
-                             varnames=[],
-                             filename=rffi.charp2str(filename),
-                             name=rffi.charp2str(funcname),
-                             firstlineno=rffi.cast(lltype.Signed, firstlineno),
-                             lnotab="",
-                             freevars=[],
-                             cellvars=[]))
+    return PyCode(space,
+                  argcount=0,
+                  kwonlyargcount=0,
+                  nlocals=0,
+                  stacksize=0,
+                  flags=0,
+                  code="",
+                  consts=[],
+                  names=[],
+                  varnames=[],
+                  filename=rffi.charp2str(filename),
+                  name=rffi.charp2str(funcname),
+                  firstlineno=rffi.cast(lltype.Signed, firstlineno),
+                  lnotab="",
+                  freevars=[],
+                  cellvars=[]))
 
 @cpython_api([PyCodeObject], Py_ssize_t, error=CANNOT_FAIL)
 def PyCode_GetNumFree(space, w_co):
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -19,35 +19,35 @@
     # Get the builtins from current globals
     if caller is not None:
         w_globals = caller.get_w_globals()
-        w_builtin = space.getitem(w_globals, space.wrap('__builtins__'))
+        w_builtin = space.getitem(w_globals, space.newtext('__builtins__'))
     else:
         # No globals -- use standard builtins, and fake globals
         w_builtin = space.getbuiltinmodule('builtins')
         w_globals = space.newdict()
-        space.setitem(w_globals, space.wrap("__builtins__"), w_builtin)
+        space.setitem(w_globals, space.newtext("__builtins__"), w_builtin)
 
     # Get the __import__ function from the builtins
     if space.isinstance_w(w_builtin, space.w_dict):
-        w_import = space.getitem(w_builtin, space.wrap("__import__"))
+        w_import = space.getitem(w_builtin, space.newtext("__import__"))
     else:
-        w_import = space.getattr(w_builtin, space.wrap("__import__"))
+        w_import = space.getattr(w_builtin, space.newtext("__import__"))
 
     # Call the __import__ function with the proper argument list
     # Always use absolute import here.
     return space.call_function(w_import,
                                w_name, w_globals, w_globals,
-                               space.newlist([space.wrap("__doc__")]))
+                               space.newlist([space.newtext("__doc__")]))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyImport_ImportModule(space, name):
-    return PyImport_Import(space, space.wrap(rffi.charp2str(name)))
+    return PyImport_Import(space, space.newtext(rffi.charp2str(name)))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyImport_ImportModuleNoBlock(space, name):
     space.warn(
-        space.wrap('PyImport_ImportModuleNoBlock() is not non-blocking'),
+        space.newtext('PyImport_ImportModuleNoBlock() is not non-blocking'),
         space.w_RuntimeWarning)
-    return PyImport_Import(space, space.wrap(rffi.charp2str(name)))
+    return PyImport_Import(space, space.newtext(rffi.charp2str(name)))
 
 
 @cts.decl(
@@ -75,7 +75,7 @@
 @cpython_api([PyObject], PyObject)
 def PyImport_ReloadModule(space, w_mod):
     w_import = space.builtin.get('__import__')
-    w_imp = space.call_function(w_import, space.wrap('imp'))
+    w_imp = space.call_function(w_import, space.newtext('imp'))
     return space.call_method(w_imp, 'reload', w_mod)
 
 @cpython_api([CONST_STRING], PyObject, result_borrowed=True)
@@ -95,8 +95,8 @@
     modulename = rffi.charp2str(name)
     w_mod = check_sys_modules_w(space, modulename)
     if not w_mod or space.is_w(w_mod, space.w_None):
-        w_mod = Module(space, space.wrap(modulename))
-    space.setitem(space.sys.get('modules'), space.wrap(modulename), w_mod)
+        w_mod = Module(space, space.newtext(modulename))
+    space.setitem(space.sys.get('modules'), space.newtext(modulename), w_mod)
     # return a borrowed ref --- assumes one copy in sys.modules
     return w_mod
 
@@ -139,13 +139,13 @@
     """Like PyImport_ExecCodeModule(), but the __file__ attribute of
     the module object is set to pathname if it is non-NULL."""
     code = space.interp_w(PyCode, w_code)
-    w_name = space.wrap(rffi.charp2str(name))
+    w_name = space.newtext(rffi.charp2str(name))
     if pathname:
         pathname = rffi.charp2str(pathname)
     else:
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
-    space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
+    space.setattr(w_mod, space.newtext('__file__'), space.wrap_fsdecoded(pathname))
     cpathname = importing.make_compiled_pathname(pathname)
     importing.exec_code_module(space, w_mod, code, pathname, cpathname)
     return w_mod
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -40,7 +40,7 @@
 def PyIter_Check(space, w_obj):
     """Return true if the object o supports the iterator protocol."""
     try:
-        w_attr = space.getattr(space.type(w_obj), space.wrap("__next__"))
+        w_attr = space.getattr(space.type(w_obj), space.newtext("__next__"))
     except:
         return False
     else:
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -86,7 +86,7 @@
     """Insert the item item into list list in front of index index.  Return
     0 if successful; return -1 and set an exception if unsuccessful.
     Analogous to list.insert(index, item)."""
-    space.call_method(space.w_list, "insert", w_list, space.wrap(index), w_item)
+    space.call_method(space.w_list, "insert", w_list, space.newint(index), w_item)
     return 0
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
@@ -135,8 +135,8 @@
     and high.  Return NULL and set an exception if unsuccessful.  Analogous
     to list[low:high].  Negative indices, as when slicing from Python, are not
     supported."""
-    w_start = space.wrap(low)
-    w_stop = space.wrap(high)
+    w_start = space.newint(low)
+    w_stop = space.newint(high)
     return space.getslice(w_list, w_start, w_stop)
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
@@ -146,8 +146,8 @@
     be NULL, indicating the assignment of an empty list (slice deletion).
     Return 0 on success, -1 on failure.  Negative indices, as when
     slicing from Python, are not supported."""
-    w_start = space.wrap(low)
-    w_stop = space.wrap(high)
+    w_start = space.newint(low)
+    w_stop = space.newint(high)
     if w_sequence:
         space.setslice(w_list, w_start, w_stop, w_sequence)
     else:
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -137,7 +137,7 @@
     except OperationError as e:
         if not e.match(space, space.w_OverflowError):
             raise
-    if space.is_true(space.gt(w_long, space.wrap(0))):
+    if space.is_true(space.gt(w_long, space.newint(0))):
         overflow_ptr[0] = rffi.cast(rffi.INT_real, 1)
     else:
         overflow_ptr[0] = rffi.cast(rffi.INT_real, -1)
@@ -158,7 +158,7 @@
     except OperationError as e:
         if not e.match(space, space.w_OverflowError):
             raise
-    if space.is_true(space.gt(w_long, space.wrap(0))):
+    if space.is_true(space.gt(w_long, space.newint(0))):
         overflow_ptr[0] = rffi.cast(rffi.INT_real, 1)
     else:
         overflow_ptr[0] = rffi.cast(rffi.INT_real, -1)
@@ -167,7 +167,7 @@
 @cpython_api([lltype.Float], PyObject)
 def PyLong_FromDouble(space, val):
     """Return a new PyLongObject object from v, or NULL on failure."""
-    return space.int(space.wrap(val))
+    return space.int(space.newfloat(val))
 
 @cpython_api([PyObject], lltype.Float, error=-1.0)
 def PyLong_AsDouble(space, w_long):
@@ -188,8 +188,8 @@
     between 2 and 36, inclusive.  Leading spaces are ignored.  If there are
     no digits, ValueError will be raised."""
     s = rffi.charp2str(str)
-    w_str = space.wrap(s)
-    w_base = space.wrap(rffi.cast(lltype.Signed, base))
+    w_str = space.newtext(s)
+    w_base = space.newint(rffi.cast(lltype.Signed, base))
     if pend:
         pend[0] = rffi.ptradd(str, len(s))
     return space.call_function(space.w_int, w_str, w_base)
@@ -201,8 +201,8 @@
     string, length gives the number of characters, and base is the radix
     for the conversion.  The radix must be in the range [2, 36]; if it is
     out of range, ValueError will be raised."""
-    w_value = space.wrap(rffi.wcharpsize2unicode(u, length))
-    w_base = space.wrap(rffi.cast(lltype.Signed, base))
+    w_value = space.newunicode(rffi.wcharpsize2unicode(u, length))
+    w_base = space.newint(rffi.cast(lltype.Signed, base))
     return space.call_function(space.w_int, w_value, w_base)
 
 @cpython_api([rffi.VOIDP], PyObject)
@@ -214,7 +214,7 @@
     value = rffi.cast(ADDR, p)    # signed integer
     if value < 0:
         return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
-    return space.wrap(value)
+    return space.newint(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -82,7 +82,7 @@
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly one argument (%d given)",
                             self.name, length)
-            w_arg = space.getitem(w_args, space.wrap(0))
+            w_arg = space.getitem(w_args, space.newint(0))
             return generic_cpy_call(space, func, w_self, w_arg)
         elif flags & METH_VARARGS:
             return generic_cpy_call(space, func, w_self, w_args)
@@ -92,7 +92,7 @@
     def get_doc(self, space):
         doc = self.ml.c_ml_doc
         if doc:
-            return space.wrap(rffi.charp2str(rffi.cast(rffi.CCHARP,doc)))
+            return space.newtext(rffi.charp2str(rffi.cast(rffi.CCHARP,doc)))
         else:
             return space.w_None
 
@@ -176,9 +176,9 @@
         return self.wrapper_func(space, w_self, w_args, func_to_call)
 
     def descr_method_repr(self):
-        return self.space.wrap(u"<slot wrapper '%s' of '%s' objects>" %
-                               (self.method_name.decode('utf-8'),
-                                self.w_objclass.name.decode('utf-8')))
+        return self.space.newtext("<slot wrapper '%s' of '%s' objects>" %
+                                  (self.method_name,
+                                   self.w_objclass.name))
 
 def cwrapper_descr_call(space, w_self, __args__):
     self = space.interp_w(W_PyCWrapperObject, w_self)
@@ -187,7 +187,7 @@
     w_self = args_w[0]
     w_kw = space.newdict()
     for key, w_obj in kw_w.items():
-        space.setitem(w_kw, space.wrap(key), w_obj)
+        space.setitem(w_kw, space.newtext(key), w_obj)
     return self.call(space, w_self, w_args, w_kw)
 
 
@@ -197,7 +197,7 @@
     w_args = space.newtuple(args_w)
     w_kw = space.newdict()
     for key, w_obj in kw_w.items():
-        space.setitem(w_kw, space.wrap(key), w_obj)
+        space.setitem(w_kw, space.newtext(key), w_obj)
     ret = self.call(space, None, w_args, w_kw)
     return ret
 
@@ -208,7 +208,7 @@
     w_args = space.newtuple(args_w[1:])
     w_kw = space.newdict()
     for key, w_obj in kw_w.items():
-        space.setitem(w_kw, space.wrap(key), w_obj)
+        space.setitem(w_kw, space.newtext(key), w_obj)
     ret = self.call(space, w_instance, w_args, w_kw)
     return ret
 
@@ -216,12 +216,12 @@
     if w_obj is None or space.is_w(w_obj, space.w_None):
         return w_function
     else:
-        return space.wrap(Method(space, w_function, w_obj))
+        return Method(space, w_function, w_obj)
 
 def cclassmethod_descr_get(space, w_function, w_obj, w_cls=None):
     if not w_cls:
         w_cls = space.type(w_obj)
-    return space.wrap(Method(space, w_function, w_cls))
+    return Method(space, w_function, w_cls)
 
 
 W_PyCFunctionObject.typedef = TypeDef(
@@ -270,7 +270,7 @@
 
 @cpython_api([lltype.Ptr(PyMethodDef), PyObject, PyObject], PyObject)
 def PyCFunction_NewEx(space, ml, w_self, w_name):
-    return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
+    return W_PyCFunctionObject(space, ml, w_self, w_name)
 
 @cts.decl("PyCFunction PyCFunction_GetFunction(PyObject *)")
 def PyCFunction_GetFunction(space, w_obj):
@@ -285,19 +285,19 @@
 
 @cpython_api([PyObject], PyObject)
 def PyStaticMethod_New(space, w_func):
-    return space.wrap(StaticMethod(w_func))
+    return StaticMethod(w_func)
 
 @cpython_api([PyObject], PyObject)
 def PyClassMethod_New(space, w_func):
-    return space.wrap(ClassMethod(w_func))
+    return ClassMethod(w_func)
 
 @cpython_api([PyTypeObjectPtr, lltype.Ptr(PyMethodDef)], PyObject)
 def PyDescr_NewMethod(space, w_type, method):
-    return space.wrap(W_PyCMethodObject(space, method, w_type))
+    return W_PyCMethodObject(space, method, w_type)
 
 @cpython_api([PyObject, lltype.Ptr(PyMethodDef)], PyObject)
 def PyDescr_NewClassMethod(space, w_type, method):
-    return space.wrap(W_PyCClassMethodObject(space, method, w_type))
+    return W_PyCClassMethodObject(space, method, w_type)
 
 @cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject)
 def Py_FindMethod(space, table, w_obj, name_ptr):
@@ -321,9 +321,9 @@
                 break
             if name == "__methods__":
                 method_list_w.append(
-                    space.wrap(rffi.charp2str(rffi.cast(rffi.CCHARP, method.c_ml_name))))
+                    space.newtext(rffi.charp2str(rffi.cast(rffi.CCHARP, method.c_ml_name))))
             elif rffi.charp2str(rffi.cast(rffi.CCHARP, method.c_ml_name)) == name: # XXX expensive copy
-                return space.wrap(W_PyCFunctionObject(space, method, w_obj))
+                return W_PyCFunctionObject(space, method, w_obj)
     if name == "__methods__":
         return space.newlist(method_list_w)
-    raise OperationError(space.w_AttributeError, space.wrap(name))
+    raise OperationError(space.w_AttributeError, space.newtext(name))
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -43,7 +43,7 @@
     f_name, f_path = state.package_context
     if f_name is not None:
         modname = f_name
-    w_mod = space.wrap(Module(space, space.wrap(modname)))
+    w_mod = Module(space, space.newtext(modname))
     state.package_context = None, None
 
     if f_path is not None:
@@ -52,15 +52,15 @@
         dict_w = {}
     convert_method_defs(space, dict_w, methods, None, w_mod, modname)
     for key, w_value in dict_w.items():
-        space.setattr(w_mod, space.wrap(key), w_value)
+        space.setattr(w_mod, space.newtext(key), w_value)
     if doc:
-        space.setattr(w_mod, space.wrap("__doc__"),
-                      space.wrap(doc))
+        space.setattr(w_mod, space.newtext("__doc__"),
+                      space.newtext(doc))
     return w_mod
 
 
 def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None):
-    w_name = space.wrap(name)
+    w_name = space.newtext_or_none(name)
     methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods)
     if methods:
         i = -1
@@ -77,7 +77,7 @@
                     raise oefmt(space.w_ValueError,
                             "module functions cannot set METH_CLASS or "
                             "METH_STATIC")
-                w_obj = space.wrap(W_PyCFunctionObject(space, method, w_self, w_name))
+                w_obj = W_PyCFunctionObject(space, method, w_self, w_name)
             else:
                 if methodname in dict_w and not (flags & METH_COEXIST):
                     continue
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -123,7 +123,7 @@
     value on success, or NULL on failure. This is the equivalent of the Python
     expression o.attr_name."""
     name = rffi.charp2str(name_ptr)
-    return space.getattr(w_obj, space.wrap(name))
+    return space.getattr(w_obj, space.newtext(name))
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_HasAttr(space, w_obj, w_name):
@@ -137,7 +137,7 @@
 def PyObject_HasAttrString(space, w_obj, name_ptr):
     try:
         name = rffi.charp2str(name_ptr)
-        w_res = operation.hasattr(space, w_obj, space.wrap(name))
+        w_res = operation.hasattr(space, w_obj, space.newtext(name))
         return space.is_true(w_res)
     except OperationError:
         return 0
@@ -149,7 +149,7 @@
 
 @cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1)
 def PyObject_SetAttrString(space, w_obj, name_ptr, w_value):
-    w_name = space.wrap(rffi.charp2str(name_ptr))
+    w_name = space.newtext(rffi.charp2str(name_ptr))
     operation.setattr(space, w_obj, w_name, w_value)
     return 0
 
@@ -164,7 +164,7 @@
 def PyObject_DelAttrString(space, w_obj, name_ptr):
     """Delete attribute named attr_name, for object o. Returns -1 on failure.
     This is the equivalent of the Python statement del o.attr_name."""
-    w_name = space.wrap(rffi.charp2str(name_ptr))
+    w_name = space.newtext(rffi.charp2str(name_ptr))
     space.delattr(w_obj, w_name)
     return 0
 
@@ -239,7 +239,7 @@
 @cpython_api([PyObject], PyObject)
 def PyObject_Str(space, w_obj):
     if w_obj is None:
-        return space.wrap("<NULL>")
+        return space.newtext("<NULL>")
     return space.str(w_obj)
 
 @cts.decl("PyObject * PyObject_Bytes(PyObject *v)")
@@ -262,13 +262,13 @@
     Python expression repr(o).  Called by the repr() built-in function and
     by reverse quotes."""
     if w_obj is None:
-        return space.wrap("<NULL>")
+        return space.newtext("<NULL>")
     return space.repr(w_obj)
 
 @cpython_api([PyObject, PyObject], PyObject)
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
-        w_format_spec = space.wrap('')
+        w_format_spec = space.newtext('')
     w_ret = space.call_method(w_obj, '__format__', w_format_spec)
     if space.isinstance_w(w_format_spec, space.w_unicode):
         return space.unicode_from_object(w_ret)
@@ -290,7 +290,7 @@
     the Python expression unicode(o).  Called by the unicode() built-in
     function."""
     if w_obj is None:
-        return space.wrap(u"<NULL>")
+        return space.newunicode(u"<NULL>")
     return space.call_function(space.w_unicode, w_obj)
 
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
@@ -394,7 +394,7 @@
         fd = space.int_w(w_obj)
     except OperationError:
         try:
-            w_meth = space.getattr(w_obj, space.wrap('fileno'))
+            w_meth = space.getattr(w_obj, space.newtext('fileno'))
         except OperationError:
             raise oefmt(space.w_TypeError,
                         "argument must be an int, or have a fileno() method.")
@@ -418,7 +418,7 @@
 
 @cpython_api([rffi.DOUBLE], rffi.LONG, error=-1)
 def _Py_HashDouble(space, v):
-    return space.int_w(space.hash(space.wrap(v)))
+    return space.int_w(space.hash(space.newfloat(v)))
 
 @cpython_api([PyObject], lltype.Signed, error=-1)
 def PyObject_HashNotImplemented(space, o):
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -21,7 +21,7 @@
 @cpython_api([PyObject, CONST_STRING], lltype.Void)
 def PyErr_SetString(space, w_type, message_ptr):
     message = rffi.charp2str(message_ptr)
-    PyErr_SetObject(space, w_type, space.wrap(message))
+    PyErr_SetObject(space, w_type, space.newtext(message))
 
 @cpython_api([PyObject], lltype.Void, error=CANNOT_FAIL)
 def PyErr_SetNone(space, w_type):
@@ -54,7 +54,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+        ptraceback[0] = make_ref(space, operror.get_traceback())
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
@@ -166,13 +166,13 @@
     msg = _strerror(errno)
     if w_value:
         w_error = space.call_function(w_type,
-                                      space.wrap(errno),
-                                      space.wrap(msg),
+                                      space.newint(errno),
+                                      space.newtext(msg),
                                       w_value)
     else:
         w_error = space.call_function(w_type,
-                                      space.wrap(errno),
-                                      space.wrap(msg))
+                                      space.newint(errno),
+                                      space.newtext(msg))
     raise OperationError(w_type, w_error)
 
 @cpython_api([], rffi.INT_real, error=-1)
@@ -250,11 +250,11 @@
     documentation.  There is no C API for warning control."""
     if w_category is None:
         w_category = space.w_None
-    w_message = space.wrap(rffi.charp2str(message_ptr))
-    w_stacklevel = space.wrap(rffi.cast(lltype.Signed, stacklevel))
+    w_message = space.newtext(rffi.charp2str(message_ptr))
+    w_stacklevel = space.newint(rffi.cast(lltype.Signed, stacklevel))
 
-    w_module = PyImport_Import(space, space.wrap("warnings"))
-    w_warn = space.getattr(w_module, space.wrap("warn"))
+    w_module = PyImport_Import(space, space.newtext("warnings"))
+    w_warn = space.getattr(w_module, space.newtext("warn"))
     space.call_function(w_warn, w_message, w_category, w_stacklevel)
     return 0
 
@@ -284,7 +284,7 @@
     operror.normalize_exception(space)
     w_type = operror.w_type
     w_value = operror.get_w_value(space)
-    w_tb = space.wrap(operror.get_traceback())
+    w_tb = operror.get_traceback()
 
     if rffi.cast(lltype.Signed, set_sys_last_vars):
         space.sys.setdictvalue(space, "last_type", w_type)
@@ -315,10 +315,10 @@
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
 def PyTraceBack_Print(space, w_tb, w_file):
-    space.call_method(w_file, "write", space.wrap(
+    space.call_method(w_file, "write", space.newtext(
         'Traceback (most recent call last):\n'))
     w_traceback = space.call_method(space.builtin, '__import__',
-                                    space.wrap("traceback"))
+                                    space.newtext("traceback"))
     space.call_method(w_traceback, "print_tb", w_tb, space.w_None, w_file)
     return 0
 
@@ -371,7 +371,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+        ptraceback[0] = make_ref(space, operror.get_traceback())
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -17,15 +17,15 @@
     0, however, one line is read regardless of length, but EOFError is
     raised if the end of the file is reached immediately."""
     try:
-        w_readline = space.getattr(w_obj, space.wrap('readline'))
+        w_readline = space.getattr(w_obj, space.newtext('readline'))
     except OperationError:
         raise oefmt(space.w_TypeError,
             "argument must be a file, or have a readline() method.")
 
     n = rffi.cast(lltype.Signed, n)
-    if space.is_true(space.gt(space.wrap(n), space.wrap(0))):
-        return space.call_function(w_readline, space.wrap(n))
-    elif space.is_true(space.lt(space.wrap(n), space.wrap(0))):
+    if space.is_true(space.gt(space.newint(n), space.newint(0))):
+        return space.call_function(w_readline, space.newint(n))
+    elif space.is_true(space.lt(space.newint(n), space.newint(0))):
         return space.call_function(w_readline)
     else:
         # XXX Raise EOFError as specified
@@ -38,7 +38,7 @@
     filename, with a file mode given by mode, where mode has the same
     semantics as the standard C routine fopen().  On failure, return NULL."""
     w_filename = space.newbytes(rffi.charp2str(filename))
-    w_mode = space.wrap(rffi.charp2str(mode))
+    w_mode = space.newtext(rffi.charp2str(mode))
     return space.call_method(space.builtin, 'open', w_filename, w_mode)
 
 @cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject)
@@ -58,7 +58,7 @@
 def PyFile_WriteString(space, s, w_p):
     """Write string s to file object p.  Return 0 on success or -1 on
     failure; the appropriate exception will be set."""
-    w_str = space.wrap(rffi.charp2str(s))
+    w_str = space.newtext(rffi.charp2str(s))
     space.call_method(w_p, "write", w_str)
     return 0
 
@@ -79,5 +79,5 @@
 @cpython_api([PyObject], PyObject)
 def PyFile_Name(space, w_p):
     """Return the name of the file specified by p as a string object."""
-    w_name = space.getattr(w_p, space.wrap("name"))
+    w_name = space.getattr(w_p, space.newtext("name"))
     return w_name     # borrowed ref, should be a W_StringObject from the file
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -19,7 +19,7 @@
     """Return the result of repeating sequence object o count times, or NULL on
     failure.  This is the equivalent of the Python expression o * count.
     """
-    return space.mul(w_obj, space.wrap(count))
+    return space.mul(w_obj, space.newint(count))
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PySequence_Check(space, w_obj):
@@ -55,7 +55,7 @@
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
     except OperationError as e:
         if e.match(space, space.w_TypeError):
-            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+            raise OperationError(space.w_TypeError, space.newtext(rffi.charp2str(m)))
         raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
@@ -107,20 +107,20 @@
 def PySequence_GetSlice(space, w_obj, start, end):
     """Return the slice of sequence object o between i1 and i2, or NULL on
     failure. This is the equivalent of the Python expression o[i1:i2]."""
-    return space.getslice(w_obj, space.wrap(start), space.wrap(end))
+    return space.getslice(w_obj, space.newint(start), space.newint(end))
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PySequence_SetSlice(space, w_obj, start, end, w_value):
     """Assign the sequence object v to the slice in sequence object o from i1 to
     i2.  This is the equivalent of the Python statement o[i1:i2] = v."""
-    space.setslice(w_obj, space.wrap(start), space.wrap(end), w_value)
+    space.setslice(w_obj, space.newint(start), space.newint(end), w_value)
     return 0
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1)
 def PySequence_DelSlice(space, w_obj, start, end):
     """Delete the slice in sequence object o from i1 to i2.  Returns -1 on
     failure.  This is the equivalent of the Python statement del o[i1:i2]."""
-    space.delslice(w_obj, space.wrap(start), space.wrap(end))
+    space.delslice(w_obj, space.newint(start), space.newint(end))
     return 0
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject)
@@ -132,7 +132,7 @@
 
     This function used an int type for i. This might require
     changes in your code for properly supporting 64-bit systems."""
-    return space.getitem(w_obj, space.wrap(i))
+    return space.getitem(w_obj, space.newint(i))
 
 @cpython_api([PyObject, Py_ssize_t], PyObject)
 def PySequence_GetItem(space, w_obj, i):
@@ -175,7 +175,7 @@
 
     This function used an int type for count. This might require
     changes in your code for properly supporting 64-bit systems."""
-    return space.inplace_mul(w_o, space.wrap(count))
+    return space.inplace_mul(w_o, space.newint(count))
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
@@ -203,14 +203,14 @@
     if PyDict_Check(space, w_o) or not PySequence_Check(space, w_o):
         raise oefmt(space.w_TypeError,
                     "'%T' object does not support item assignment", w_o)
-    space.setitem(w_o, space.wrap(i), w_v)
+    space.setitem(w_o, space.newint(i), w_v)
     return 0
 
 @cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1)
 def PySequence_DelItem(space, w_o, i):
     """Delete the ith element of object o.  Returns -1 on failure.  This is the
     equivalent of the Python statement del o[i]."""
-    space.delitem(w_o, space.wrap(i))
+    space.delitem(w_o, space.newint(i))
     return 0
 
 @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -139,7 +139,7 @@
     res = rffi.cast(lltype.Signed, res)
     if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(bool(res))
+    return space.newbool(bool(res))
 
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
@@ -228,7 +228,7 @@
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
     check_num_args(space, w_args, 0)
-    return space.wrap(generic_cpy_call(space, func_len, w_self))
+    return space.newint(generic_cpy_call(space, func_len, w_self))
 
 def wrap_sq_item(space, w_self, w_args, func):
     func_target = rffi.cast(ssizeargfunc, func)
@@ -265,7 +265,7 @@
     res = rffi.cast(lltype.Signed, res)
     if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(bool(res))
+    return space.newbool(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -308,7 +308,7 @@
     res = generic_cpy_call(space, func_target, w_self)
     if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.newint(res)
 
 class CPyBuffer(Buffer):
     # Similar to Py_buffer
@@ -499,7 +499,7 @@
                     "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'",
                     w_self, w_self, w_other)
 
-    return space.wrap(generic_cpy_call(space, func_target, w_self, w_other))
+    return space.newint(generic_cpy_call(space, func_target, w_self, w_other))
 
 from rpython.rlib.nonconst import NonConstant
 
@@ -601,7 +601,7 @@
             @slot_function([PyObject, Py_ssize_t], PyObject)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, arg):
-                return space.call_function(slot_fn, w_self, space.wrap(arg))
+                return space.call_function(slot_fn, w_self, space.newint(arg))
             handled = True
 
     # ternary functions
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -123,7 +123,7 @@
             space = self.space
             argv = space.sys.get('argv')
             if space.len_w(argv):
-                argv0 = space.getitem(argv, space.wrap(0))
+                argv0 = space.getitem(argv, space.newint(0))
                 progname = space.unicode_w(argv0)
             else:
                 progname = u"pypy"
diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py
--- a/pypy/module/cpyext/structmember.py
+++ b/pypy/module/cpyext/structmember.py
@@ -51,13 +51,15 @@
         if typ == member_type:
             result = rffi.cast(rffi.CArrayPtr(lltyp), addr)
             if lltyp is rffi.FLOAT:
-                w_result = space.wrap(lltype.cast_primitive(lltype.Float,
+                w_result = space.newfloat(lltype.cast_primitive(lltype.Float,
                                                             result[0]))
             elif typ == T_BOOL:
                 x = rffi.cast(lltype.Signed, result[0])
-                w_result = space.wrap(x != 0)
+                w_result = space.newbool(x != 0)
+            elif typ == T_DOUBLE:
+                w_result = space.newfloat(result[0])
             else:
-                w_result = space.wrap(result[0])
+                w_result = space.newint(result[0])
             return w_result
 
     if member_type == T_STRING:
@@ -83,7 +85,7 @@
         if obj_ptr[0]:
             w_result = from_ref(space, obj_ptr[0])
         else:
-            w_name = space.wrap(rffi.charp2str(w_member.c_name))
+            w_name = space.newtext(rffi.charp2str(w_member.c_name))
             raise OperationError(space.w_AttributeError, w_name)
     else:
         raise oefmt(space.w_SystemError, "bad memberdescr type")
@@ -105,7 +107,7 @@
     elif w_value is None:
         if member_type == T_OBJECT_EX:
             if not rffi.cast(PyObjectP, addr)[0]:
-                w_name = space.wrap(rffi.charp2str(w_member.c_name))
+                w_name = space.newtext(rffi.charp2str(w_member.c_name))
                 raise OperationError(space.w_AttributeError, w_name)
         elif member_type != T_OBJECT:
             raise oefmt(space.w_TypeError,
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -210,4 +210,4 @@
     """Take a slice of the tuple pointed to by p from low to high and return it
     as a new tuple.
     """
-    return space.getslice(w_obj, space.wrap(low), space.wrap(high))
+    return space.getslice(w_obj, space.newint(low), space.newint(high))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -472,9 +472,9 @@
     set."""
     if not encoding:
         raise oefmt(space.w_TypeError, "decoding Unicode is not supported")
-    w_encoding = space.wrap(rffi.charp2str(encoding))
+    w_encoding = space.newtext(rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
     else:
         w_errors = None
 
@@ -484,7 +484,7 @@
         w_meth = None
     else:
         try:
-            w_meth = space.getattr(w_obj, space.wrap('decode'))
+            w_meth = space.getattr(w_obj, space.newtext('decode'))
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
                 raise
@@ -585,7 +585,7 @@
 def PyUnicode_FromString(space, s):
     """Create a Unicode object from an UTF-8 encoded null-terminated char buffer"""
     w_str = space.newbytes(rffi.charp2str(s))
-    return space.call_method(w_str, 'decode', space.wrap("utf-8"))
+    return space.call_method(w_str, 'decode', space.newtext("utf-8"))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyUnicode_InternFromString(space, s):
@@ -618,7 +618,7 @@
     The ordinal must be in range(0x10000) on narrow Python builds
     (UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
     raised in case it is not."""
-    w_ordinal = space.wrap(rffi.cast(lltype.Signed, ordinal))
+    w_ordinal = space.newint(rffi.cast(lltype.Signed, ordinal))
     return space.call_function(space.builtin.get('chr'), w_ordinal)
 
 @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1)
@@ -665,10 +665,10 @@
         """
         w_s = space.newbytes(rffi.charpsize2str(s, size))
         if errors:
-            w_errors = space.wrap(rffi.charp2str(errors))
+            w_errors = space.newtext(rffi.charp2str(errors))
         else:
             w_errors = None
-        return space.call_method(w_s, 'decode', space.wrap(encoding), w_errors)
+        return space.call_method(w_s, 'decode', space.newtext(encoding), w_errors)
     globals()['PyUnicode_Decode%s' % suffix] = PyUnicode_DecodeXXX
 
     @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject)
@@ -677,12 +677,12 @@
         """Encode the Py_UNICODE buffer of the given size and return a
         Python string object.  Return NULL if an exception was raised
         by the codec."""
-        w_u = space.wrap(rffi.wcharpsize2unicode(s, size))
+        w_u = space.newunicode(rffi.wcharpsize2unicode(s, size))
         if errors:
-            w_errors = space.wrap(rffi.charp2str(errors))
+            w_errors = space.newtext(rffi.charp2str(errors))
         else:
             w_errors = None
-        return space.call_method(w_u, 'encode', space.wrap(encoding), w_errors)
+        return space.call_method(w_u, 'encode', space.newtext(encoding), w_errors)
     globals()['PyUnicode_Encode%s' % suffix] = PyUnicode_EncodeXXX
 
 make_conversion_functions('UTF8', 'utf-8')
@@ -743,7 +743,7 @@
     if pbyteorder is not None:
         pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
 
-    return space.wrap(result)
+    return space.newunicode(result)
 
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
 def PyUnicode_DecodeUTF32(space, s, size, llerrors, pbyteorder):
@@ -799,7 +799,7 @@
     if pbyteorder is not None:
         pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
 
-    return space.wrap(result)
+    return space.newunicode(result)
 
 @cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP],
              rffi.INT_real, error=-1)
@@ -915,7 +915,7 @@
     return the resulting Unicode object. maxcount == -1 means replace all
     occurrences."""
     return space.call_method(w_str, "replace", w_substr, w_replstr,
-                             space.wrap(maxcount))
+                             space.newint(maxcount))
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
              rffi.INT_real, error=-1)
@@ -935,7 +935,7 @@
     """Return the number of non-overlapping occurrences of substr in
     str[start:end].  Return -1 if an error occurred."""
     w_count = space.call_method(w_str, "count", w_substr,
-                                space.wrap(start), space.wrap(end))
+                                space.newint(start), space.newint(end))
     return space.int_w(w_count)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
@@ -949,10 +949,10 @@
     has been set."""
     if rffi.cast(lltype.Signed, direction) > 0:
         w_pos = space.call_method(w_str, "find", w_substr,
-                                  space.wrap(start), space.wrap(end))
+                                  space.newint(start), space.newint(end))
     else:
         w_pos = space.call_method(w_str, "rfind", w_substr,
-                                  space.wrap(start), space.wrap(end))
+                                  space.newint(start), space.newint(end))
     return space.int_w(w_pos)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
@@ -964,7 +964,7 @@
     are not included in the resulting list."""
     if w_sep is None:
         w_sep = space.w_None
-    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+    return space.call_method(w_str, "split", w_sep, space.newint(maxsplit))
 
 @cpython_api([PyObject, rffi.INT_real], PyObject)
 def PyUnicode_Splitlines(space, w_str, keepend):
@@ -972,4 +972,4 @@
     Unicode strings.  CRLF is considered to be one line break.  If
     keepend is 0, the Line break characters are not included in the
     resulting strings."""
-    return space.call_method(w_str, "splitlines", space.wrap(keepend))
+    return space.call_method(w_str, "splitlines", space.newbool(bool(keepend)))
diff --git a/pypy/module/marshal/__init__.py b/pypy/module/marshal/__init__.py
--- a/pypy/module/marshal/__init__.py
+++ b/pypy/module/marshal/__init__.py
@@ -14,5 +14,5 @@
         'dumps'   : 'interp_marshal.dumps',
         'load'    : 'interp_marshal.load',
         'loads'   : 'interp_marshal.loads',
-        'version' : 'space.wrap(interp_marshal.Py_MARSHAL_VERSION)',
+        'version' : 'space.newint(interp_marshal.Py_MARSHAL_VERSION)',
     }
diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py
--- a/pypy/module/termios/interp_termios.py
+++ b/pypy/module/termios/interp_termios.py
@@ -44,12 +44,12 @@
     except OSError as e:
         raise convert_error(space, e)
     iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tup
-    l_w = [space.wrap(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]]
+    l_w = [space.newint(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]]
     # last one need to be chosen carefully
     cc_w = [space.newbytes(i) for i in cc]
     if lflag & rtermios.ICANON:
-        cc_w[rtermios.VMIN] = space.wrap(ord(cc[rtermios.VMIN][0]))
-        cc_w[rtermios.VTIME] = space.wrap(ord(cc[rtermios.VTIME][0]))
+        cc_w[rtermios.VMIN] = space.newint(ord(cc[rtermios.VMIN][0]))
+        cc_w[rtermios.VTIME] = space.newint(ord(cc[rtermios.VTIME][0]))
     w_cc = space.newlist(cc_w)
     l_w.append(w_cc)
     return space.newlist(l_w)
diff --git a/pypy/module/thread/error.py b/pypy/module/thread/error.py
--- a/pypy/module/thread/error.py
+++ b/pypy/module/thread/error.py
@@ -2,4 +2,4 @@
 
 def wrap_thread_error(space, msg):
     w_error = space.w_RuntimeError    # since CPython 3.3
-    return OperationError(w_error, space.wrap(msg))
+    return OperationError(w_error, space.newtext(msg))
diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py
--- a/pypy/module/thread/os_local.py
+++ b/pypy/module/thread/os_local.py
@@ -52,7 +52,7 @@
         # call __init__
         try:
             w_type = space.type(self)
-            w_init = space.getattr(w_type, space.wrap("__init__"))
+            w_init = space.getattr(w_type, space.newtext("__init__"))
             space.call_obj_args(w_init, self, self.initargs)
         except:
             # failed, forget w_dict and propagate the exception
@@ -77,7 +77,7 @@
     def descr_local__new__(space, w_subtype, __args__):
         local = space.allocate_instance(Local, w_subtype)
         Local.__init__(local, space, __args__)
-        return space.wrap(local)
+        return local
 
     def descr_local__init__(self, space):
         # No arguments allowed
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -155,7 +155,7 @@
 def allocate_lock(space):
     """Create a new lock object.  (allocate() is an obsolete synonym.)
 See LockType.__doc__ for information about locks."""
-    return space.wrap(Lock(space))
+    return Lock(space)
 
 def _set_sentinel(space):
     """_set_sentinel() -> lock
@@ -169,7 +169,7 @@
     # after forking the lock must be recreated! forget the old lock
     lock = Lock(space)
     ec._sentinel_lock = lock
-    return space.wrap(lock)
+    return lock
 
 class W_RLock(W_Root):
     def __init__(self, space):
@@ -183,7 +183,7 @@
     def descr__new__(space, w_subtype):
         self = space.allocate_instance(W_RLock, w_subtype)
         W_RLock.__init__(self, space)
-        return space.wrap(self)
+        return self
 
     def descr__repr__(self, space):
         classname = space.getfulltypename(self)
@@ -229,7 +229,7 @@
             self.rlock_owner = tid
             self.rlock_count = 1
 
-        return space.wrap(r)
+        return space.newbool(r)
 
     def release_w(self, space):
         """Release the lock, allowing another thread that is blocked waiting for
@@ -280,7 +280,7 @@
         count, self.rlock_count = self.rlock_count, 0
         owner, self.rlock_owner = self.rlock_owner, 0
         self.lock.release()
-        return space.newtuple([space.wrap(count), space.wrap(owner)])
+        return space.newtuple([space.newint(count), space.newint(owner)])
 
     def descr__enter__(self, space):
         self.acquire_w(space)
diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py
--- a/pypy/module/thread/test/test_local.py
+++ b/pypy/module/thread/test/test_local.py
@@ -122,6 +122,7 @@
             return {}
         def wrap(self, obj):
             return obj
+        newtext = wrap
         def type(self, obj):
             return type(obj)
         class config:

From pypy.commits at gmail.com  Tue Feb 14 11:05:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 08:05:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge a3f02433a915
Message-ID: <58a32ab1.018e1c0a.260b6.9ab4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90113:6c401ebf28ef
Date: 2017-02-14 17:04 +0100
http://bitbucket.org/pypy/pypy/changeset/6c401ebf28ef/

Log:	hg merge a3f02433a915

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -464,27 +464,27 @@
         return space.wrap(h)
 
     def get_count(self, space):
-        return space.wrap(self.count)
+        return space.newint(self.count)
 
     def _ismine(self):
         return self.count > 0 and rthread.get_ident() == self.last_tid
 
     def is_mine(self, space):
-        return space.wrap(self._ismine())
+        return space.newbool(self._ismine())
 
     def is_zero(self, space):
         try:
             res = semlock_iszero(self, space)
         except OSError as e:
             raise wrap_oserror(space, e)
-        return space.wrap(res)
+        return space.newbool(res)
 
     def get_value(self, space):
         try:
             val = semlock_getvalue(self, space)
         except OSError as e:
             raise wrap_oserror(space, e)
-        return space.wrap(val)
+        return space.newint(val)
 
     @unwrap_spec(block=bool)
     def acquire(self, space, block=True, w_timeout=None):
@@ -540,7 +540,7 @@
         #
         self = space.allocate_instance(W_SemLock, w_cls)
         self.__init__(space, handle, kind, maxvalue, name)
-        return space.wrap(self)
+        return self
 
     def enter(self, space):
         return self.acquire(space, w_timeout=space.w_None)
@@ -570,7 +570,7 @@
     self = space.allocate_instance(W_SemLock, w_subtype)
     self.__init__(space, handle, kind, maxvalue, name)
 
-    return space.wrap(self)
+    return self
 
 W_SemLock.typedef = TypeDef(
     "SemLock",
diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py
--- a/pypy/module/pwd/interp_pwd.py
+++ b/pypy/module/pwd/interp_pwd.py
@@ -73,15 +73,15 @@
 
 def make_struct_passwd(space, pw):
     w_passwd_struct = space.getattr(space.getbuiltinmodule('pwd'),
-                                    space.wrap('struct_passwd'))
+                                    space.newtext('struct_passwd'))
     w_tuple = space.newtuple([
-        space.wrap(rffi.charp2str(pw.c_pw_name)),
-        space.wrap(rffi.charp2str(pw.c_pw_passwd)),
-        space.int(space.wrap(pw.c_pw_uid)),
-        space.int(space.wrap(pw.c_pw_gid)),
-        space.wrap(rffi.charp2str(pw.c_pw_gecos)),
-        space.wrap(rffi.charp2str(pw.c_pw_dir)),
-        space.wrap(rffi.charp2str(pw.c_pw_shell)),
+        space.newtext(rffi.charp2str(pw.c_pw_name)),
+        space.newtext(rffi.charp2str(pw.c_pw_passwd)),
+        space.int(space.newint(pw.c_pw_uid)),
+        space.int(space.newint(pw.c_pw_gid)),
+        space.newtext(rffi.charp2str(pw.c_pw_gecos)),
+        space.newtext(rffi.charp2str(pw.c_pw_dir)),
+        space.newtext(rffi.charp2str(pw.c_pw_shell)),
         ])
     return space.call_function(w_passwd_struct, w_tuple)
 
@@ -102,7 +102,7 @@
         raise
     pw = c_getpwuid(uid)
     if not pw:
-        raise OperationError(space.w_KeyError, space.wrap(
+        raise OperationError(space.w_KeyError, space.newtext(
             "%s: %d" % (msg, widen(uid))))
     return make_struct_passwd(space, pw)
 
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -55,16 +55,17 @@
         except KeyError:
             raise OperationError(space.w_KeyError, space.wrap_fsdecoded(name))
         assert isinstance(w_zipimporter, W_ZipImporter)
-        w = space.wrap
         w_fs = space.wrap_fsdecoded
         w_d = space.newdict()
         for key, info in w_zipimporter.zip_file.NameToInfo.iteritems():
             if ZIPSEP != os.path.sep:
                 key = key.replace(ZIPSEP, os.path.sep)
             space.setitem(w_d, w_fs(key), space.newtuple([
-                w_fs(info.filename), w(info.compress_type), w(info.compress_size),
-                w(info.file_size), w(info.file_offset), w(info.dostime),
-                w(info.dosdate), w(info.CRC)]))
+                w_fs(info.filename), space.newint(info.compress_type),
+                space.newint(info.compress_size),
+                space.newint(info.file_size), space.newint(info.file_offset),
+                space.newint(info.dostime),
+                space.newint(info.dosdate), space.newint(info.CRC)]))
         return w_d
 
     def keys(self, space):
@@ -148,10 +149,9 @@
             return fname
 
     def import_py_file(self, space, modname, filename, buf, pkgpath):
-        w = space.wrap
-        w_mod = w(Module(space, space.wrap_fsdecoded(modname)))
+        w_mod = Module(space, space.wrap_fsdecoded(modname))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
-        space.setattr(w_mod, w('__loader__'), space.wrap(self))
+        space.setattr(w_mod, space.newtext('__loader__'), self)
         importing._prepare_module(space, w_mod, real_name, pkgpath)
         co_filename = self.make_co_filename(filename)
         code_w = importing.parse_source_module(space, co_filename, buf)
@@ -159,7 +159,6 @@
         return w_mod
 
     def _parse_mtime(self, space, filename):
-        w = space.wrap
         try:
             info = self.zip_file.NameToInfo[filename]
             t = info.date_time
@@ -167,13 +166,15 @@
             return 0
         else:
             w_mktime = space.getattr(space.getbuiltinmodule('time'),
-                                     w('mktime'))
+                                     space.newtext('mktime'))
             # XXX this is incredible fishing around module limitations
             #     in order to compare timestamps of .py and .pyc files
             # we need time.mktime support on rpython level
+            w = space.newint
             all = [w(t[0]), w(t[1]), w(t[2]), w(t[3]), w(t[4]),
                    w(t[5]), w(0), w(1), w(-1)]
-            mtime = int(space.float_w(space.call_function(w_mktime, space.newtuple(all))))
+            mtime = int(space.float_w(space.call_function(
+                w_mktime, space.newtuple(all))))
             return mtime
 
     def check_newer_pyfile(self, space, filename, timestamp):
@@ -198,18 +199,17 @@
         return True
 
     def import_pyc_file(self, space, modname, filename, buf, pkgpath):
-        w = space.wrap
         magic = importing._get_long(buf[:4])
         timestamp = importing._get_long(buf[4:8])
         if not self.can_use_pyc(space, filename, magic, timestamp):
             return None
         # zipimport ignores the size field
         buf = buf[12:] # XXX ugly copy, should use sequential read instead
-        w_mod = w(Module(space, w(modname)))
+        w_mod = Module(space, space.newtext(modname))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
-        space.setattr(w_mod, w('__loader__'), space.wrap(self))
+        space.setattr(w_mod, space.newtext('__loader__'), self)
         importing._prepare_module(space, w_mod, real_name, pkgpath)
-        result = importing.load_compiled_module(space, w(modname), w_mod,
+        result = importing.load_compiled_module(space, space.newtext(modname), w_mod,
                                                 real_name, magic, timestamp,
                                                 buf)
         return result
@@ -228,7 +228,7 @@
         filename = self.make_filename(fullname)
         for _, _, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
-                return space.wrap(self)
+                return self
 
     def make_filename(self, fullname):
         startpos = fullname.rfind('.') + 1 # 0 when not found
@@ -309,13 +309,13 @@
                                             magic, timestamp):
                         continue
                     # zipimport ignores the size field
-                    code_w = importing.read_compiled_module(
+                    w_code = importing.read_compiled_module(
                         space, filename + ext, source[12:])
                 else:
                     co_filename = self.make_co_filename(filename+ext)
-                    code_w = importing.parse_source_module(
+                    w_code = importing.parse_source_module(
                         space, co_filename, source)
-                return space.wrap(code_w)
+                return w_code
         raise oefmt(get_error(space),
                     "Cannot find source or code for %R in %R",
                     w_fullname, space.wrap_fsdecoded(self.name))
@@ -359,7 +359,7 @@
         filename = self.make_filename(fullname)
         for _, is_package, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
-                return space.wrap(is_package)
+                return space.newbool(is_package)
         raise oefmt(get_error(space),
                     "Cannot find module %R in %R",
                     space.wrap_fsdecoded(filename),
@@ -436,7 +436,7 @@
         prefix = prefix[1:]
     if prefix and not prefix.endswith(ZIPSEP) and not prefix.endswith(os.path.sep):
         prefix += ZIPSEP
-    w_result = space.wrap(W_ZipImporter(space, name, filename, zip_file, prefix))
+    w_result = W_ZipImporter(space, name, filename, zip_file, prefix)
     zip_cache.set(filename, w_result)
     return w_result
 

From pypy.commits at gmail.com  Tue Feb 14 11:37:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 08:37:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <58a33238.8ba4df0a.24a67.7867@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90114:f1c5de555036
Date: 2017-02-14 17:28 +0100
http://bitbucket.org/pypy/pypy/changeset/f1c5de555036/

Log:	fix

diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -58,7 +58,7 @@
 def PyFile_WriteString(space, s, w_p):
     """Write string s to file object p.  Return 0 on success or -1 on
     failure; the appropriate exception will be set."""
-    w_str = space.wrap(rffi.charp2str(s))
+    w_str = space.newtext(rffi.charp2str(s))
     space.call_method(w_p, "write", w_str)
     return 0
 

From pypy.commits at gmail.com  Tue Feb 14 11:37:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 08:37:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge c5c3e2fb39a6
Message-ID: <58a3323a.dea2df0a.e85df.7972@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90115:a675735c36bf
Date: 2017-02-14 17:36 +0100
http://bitbucket.org/pypy/pypy/changeset/a675735c36bf/

Log:	hg merge c5c3e2fb39a6

diff too long, truncating to 2000 out of 4500 lines

diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -27,7 +27,7 @@
             return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
         else:
             w_raw_cdata = space.call_function(self.w_alloc,
-                                              space.wrap(datasize))
+                                              space.newint(datasize))
             if not isinstance(w_raw_cdata, cdataobj.W_CData):
                 raise oefmt(space.w_TypeError,
                             "alloc() must return a cdata object (got %T)",
@@ -76,7 +76,7 @@
     if w_alloc is None and w_free is not None:
         raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
     alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
-    return space.wrap(alloc)
+    return alloc
 
 
 default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
diff --git a/pypy/module/_cffi_backend/call_python.py b/pypy/module/_cffi_backend/call_python.py
--- a/pypy/module/_cffi_backend/call_python.py
+++ b/pypy/module/_cffi_backend/call_python.py
@@ -90,7 +90,7 @@
     ffi = space.interp_w(W_FFIObject, w_ffi)
 
     if space.is_w(w_name, space.w_None):
-        w_name = space.getattr(w_python_callable, space.wrap('__name__'))
+        w_name = space.getattr(w_python_callable, space.newtext('__name__'))
     name = space.str_w(w_name)
 
     ctx = ffi.ctxobj.ctx
@@ -130,4 +130,4 @@
 
 @specialize.memo()
 def get_generic_decorator(space):
-    return space.wrap(interp2app(externpy_deco))
+    return space.wrap(interp2app(externpy_deco)) # init time
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -58,21 +58,21 @@
 
     def repr(self):
         extra2 = self._repr_extra()
-        extra1 = u''
+        extra1 = ''
         if not isinstance(self, W_CDataNewOwning):
             # it's slightly confusing to get "<cdata 'struct foo' 0x...>"
             # because the struct foo is not owned.  Trying to make it
             # clearer, write in this case "<cdata 'struct foo &' 0x...>".
             from pypy.module._cffi_backend import ctypestruct
             if isinstance(self.ctype, ctypestruct.W_CTypeStructOrUnion):
-                extra1 = u' &'
-        return self.space.wrap(u"<cdata '%s%s' %s>" % (
-            self.ctype.name.decode('utf-8'), extra1, extra2.decode('utf-8')))
+                extra1 = ' &'
+        return self.space.newtext("<cdata '%s%s' %s>" % (
+            self.ctype.name, extra1, extra2))
 
     def bool(self):
         with self as ptr:
             nonzero = self.ctype.nonzero(ptr)
-        return self.space.wrap(nonzero)
+        return self.space.newbool(nonzero)
 
     def int(self, space):
         with self as ptr:
@@ -95,7 +95,7 @@
         from pypy.module._cffi_backend import ctypearray
         space = self.space
         if isinstance(self.ctype, ctypearray.W_CTypeArray):
-            return space.wrap(self.get_array_length())
+            return space.newint(self.get_array_length())
         raise oefmt(space.w_TypeError,
                     "cdata of type '%s' has no len()", self.ctype.name)
 
@@ -137,7 +137,7 @@
         # alignment (to 4, 8, maybe 16 bytes), so we use the following
         # formula to avoid the trailing bits being always 0.
         h = h ^ (h >> 4)
-        return self.space.wrap(h)
+        return self.space.newint(h)
 
     def getitem(self, w_index):
         space = self.space
@@ -319,7 +319,7 @@
                         "pointer subtraction: the distance between the two "
                         "pointers is not a multiple of the item size")
                 diff //= itemsize
-            return space.wrap(diff)
+            return space.newint(diff)
         #
         return self._add_or_sub(w_other, -1)
 
@@ -441,7 +441,7 @@
         if isinstance(ct, W_CTypePointer):
             ct = ct.ctitem
         lst = ct.cdata_dir()
-        return space.newlist([space.wrap(s) for s in lst])
+        return space.newlist([space.newtext(s) for s in lst])
 
     def get_structobj(self):
         return None
diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py
--- a/pypy/module/_cffi_backend/cerrno.py
+++ b/pypy/module/_cffi_backend/cerrno.py
@@ -13,7 +13,7 @@
 _errno_after  = rposix._errno_after
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_alterrno())
+    return space.newint(rposix.get_saved_alterrno())
 
 @unwrap_spec(errno=int)
 def set_errno(space, errno):
@@ -27,4 +27,4 @@
     if code == -1:
         code = GetLastError_alt_saved()
     message = FormatError(code)
-    return space.newtuple([space.wrap(code), space.wrap(message)])
+    return space.newtuple([space.newint(code), space.newtext(message)])
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -39,12 +39,12 @@
     if src_ctx.c_includes:
         lib.make_includes_from(src_ctx.c_includes)
 
-    w_name = space.wrap(name)
+    w_name = space.newtext(name)
     module = Module(space, w_name)
     if path is not None:
         module.setdictvalue(space, '__file__', space.wrap_fsdecoded(path))
-    module.setdictvalue(space, 'ffi', space.wrap(ffi))
-    module.setdictvalue(space, 'lib', space.wrap(lib))
+    module.setdictvalue(space, 'ffi', ffi)
+    module.setdictvalue(space, 'lib', lib)
     w_modules_dict = space.sys.get('modules')
-    space.setitem(w_modules_dict, w_name, space.wrap(module))
-    space.setitem(w_modules_dict, space.wrap(name + '.lib'), space.wrap(lib))
+    space.setitem(w_modules_dict, w_name, space.newtext(module))
+    space.setitem(w_modules_dict, space.newtext(name + '.lib'), space.newtext(lib))
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -100,7 +100,7 @@
             return self.space.wrap(self.ctitem)
         if attrchar == 'l':     # length
             if self.length >= 0:
-                return self.space.wrap(self.length)
+                return self.space.newint(self.length)
             else:
                 return self.space.w_None
         return W_CTypePtrOrArray._fget(self, attrchar)
@@ -121,7 +121,7 @@
         self._stop = rffi.ptradd(self._next, length * ctitem.size)
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         result = self._next
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -120,11 +120,11 @@
             return self.space.newtuple([self.space.wrap(a)
                                         for a in self.fargs])
         if attrchar == 'r':    # result
-            return self.space.wrap(self.ctitem)
+            return self.ctitem
         if attrchar == 'E':    # ellipsis
             return self.space.wrap(self.ellipsis)
         if attrchar == 'A':    # abi
-            return self.space.wrap(self.abi)
+            return self.space.newint(self.abi)
         return W_CTypePtrBase._fget(self, attrchar)
 
     def call(self, funcaddr, args_w):
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -34,7 +34,7 @@
 
     def repr(self):
         space = self.space
-        return space.wrap("<ctype '%s'>" % (self.name,))
+        return space.newtext("<ctype '%s'>" % (self.name,))
 
     def extra_repr(self, cdata):
         if cdata:
@@ -229,17 +229,17 @@
     # __________ app-level attributes __________
     def dir(self):
         space = self.space
-        lst = [space.wrap(name)
+        lst = [space.newtext(name)
                   for name in _name_of_attributes
-                  if space.findattr(self, space.wrap(name)) is not None]
+                  if space.findattr(self, space.newtext(name)) is not None]
         return space.newlist(lst)
 
     def _fget(self, attrchar):
         space = self.space
         if attrchar == 'k':     # kind
-            return space.wrap(self.kind)      # class attribute
+            return space.newtext(self.kind)      # class attribute
         if attrchar == 'c':     # cname
-            return space.wrap(self.name)
+            return space.newtext(self.name)
         raise oefmt(space.w_AttributeError,
                     "ctype '%s' has no such attribute", self.name)
 
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -122,7 +122,7 @@
     _attrs_ = []
 
     def cast_to_int(self, cdata):
-        return self.space.wrap(ord(cdata[0]))
+        return self.space.newint(ord(cdata[0]))
 
     def convert_to_object(self, cdata):
         return self.space.newbytes(cdata[0])
@@ -166,11 +166,11 @@
 
     def cast_to_int(self, cdata):
         unichardata = rffi.cast(WCHAR_INTP, cdata)
-        return self.space.wrap(unichardata[0])
+        return self.space.newint(unichardata[0])
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.wrap(unichardata[0])
+        return self.space.newunicode(unichardata[0])
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
@@ -195,7 +195,7 @@
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
-        return self.space.wrap(u)
+        return self.space.newunicode(u)
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
@@ -214,14 +214,14 @@
     def convert_to_object(self, cdata):
         if self.value_fits_long:
             value = misc.read_raw_long_data(cdata, self.size)
-            return self.space.wrap(value)
+            return self.space.newint(value)
         else:
             return self._convert_to_object_longlong(cdata)
 
     def _convert_to_object_longlong(self, cdata):
         # in its own function: LONGLONG may make the whole function jit-opaque
         value = misc.read_raw_signed_data(cdata, self.size)
-        return self.space.wrap(value)    # r_longlong => on 32-bit, 'long'
+        return self.space.newint(value)    # r_longlong => on 32-bit, 'long'
 
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
@@ -271,7 +271,7 @@
                 overflowed = misc.pack_list_to_raw_array_bounds_signed(
                     int_list, cdata, self.size)
                 if overflowed != 0:
-                    self._overflow(self.space.wrap(overflowed))
+                    self._overflow(self.space.newint(overflowed))
             return True
         return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
 
@@ -316,16 +316,16 @@
         if self.value_fits_ulong:
             value = misc.read_raw_ulong_data(cdata, self.size)
             if self.value_fits_long:
-                return self.space.wrap(intmask(value))
+                return self.space.newint(intmask(value))
             else:
-                return self.space.wrap(value)    # r_uint => 'long' object
+                return self.space.newint(value)    # r_uint => 'long' object
         else:
             return self._convert_to_object_longlong(cdata)
 
     def _convert_to_object_longlong(self, cdata):
         # in its own function: LONGLONG may make the whole function jit-opaque
         value = misc.read_raw_unsigned_data(cdata, self.size)
-        return self.space.wrap(value)    # r_ulonglong => 'long' object
+        return self.space.newint(value)    # r_ulonglong => 'long' object
 
     def get_vararg_type(self):
         if self.size < rffi.sizeof(rffi.INT):
@@ -349,7 +349,7 @@
             overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
                 int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
-                self._overflow(self.space.wrap(overflowed))
+                self._overflow(self.space.newint(overflowed))
             return True
         return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
 
@@ -419,7 +419,7 @@
 
     def convert_to_object(self, cdata):
         value = misc.read_raw_float_data(cdata, self.size)
-        return self.space.wrap(value)
+        return self.space.newfloat(value)
 
     def convert_from_object(self, cdata, w_ob):
         space = self.space
@@ -499,7 +499,7 @@
 
     def float(self, cdata):
         value = self._read_from_longdouble(cdata)
-        return self.space.wrap(value)
+        return self.space.newfloat(value)
 
     def convert_to_object(self, cdata):
         w_cdata = cdataobj.W_CDataMem(self.space, self)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -176,7 +176,7 @@
                     "will be forbidden in the future (check that the types "
                     "are as you expect; use an explicit ffi.cast() if they "
                     "are correct)" % (other.name, self.name))
-                space.warn(space.wrap(msg), space.w_UserWarning)
+                space.warn(space.newtext(msg), space.w_UserWarning)
             else:
                 raise self._convert_error("compatible pointer", w_ob)
 
@@ -385,7 +385,7 @@
 
     def _fget(self, attrchar):
         if attrchar == 'i':     # item
-            return self.space.wrap(self.ctitem)
+            return self.ctitem
         return W_CTypePtrBase._fget(self, attrchar)
 
 # ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -70,8 +70,8 @@
             result = [None] * len(self._fields_list)
             for fname, field in self._fields_dict.iteritems():
                 i = self._fields_list.index(field)
-                result[i] = space.newtuple([space.wrap(fname),
-                                            space.wrap(field)])
+                result[i] = space.newtuple([space.newtext(fname),
+                                            field])
             return space.newlist(result)
         return W_CType._fget(self, attrchar)
 
@@ -93,7 +93,7 @@
         try:
             cfield = self._getcfield_const(fieldname)
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap(fieldname))
+            raise OperationError(space.w_KeyError, space.newtext(fieldname))
         if cfield.bitshift >= 0:
             raise oefmt(space.w_TypeError, "not supported for bitfields")
         return (cfield.ctype, cfield.offset)
@@ -280,14 +280,14 @@
                 shiftforsign = r_uint(1) << (self.bitsize - 1)
                 value = ((value >> self.bitshift) + shiftforsign) & valuemask
                 result = intmask(value) - intmask(shiftforsign)
-                return space.wrap(result)
+                return space.newint(result)
             else:
                 value = misc.read_raw_unsigned_data(cdata, ctype.size)
                 valuemask = (r_ulonglong(1) << self.bitsize) - 1
                 shiftforsign = r_ulonglong(1) << (self.bitsize - 1)
                 value = ((value >> self.bitshift) + shiftforsign) & valuemask
                 result = r_longlong(value) - r_longlong(shiftforsign)
-                return space.wrap(result)
+                return space.newint(result)
         #
         if isinstance(ctype, ctypeprim.W_CTypePrimitiveUnsigned):
             value_fits_long = ctype.value_fits_long
@@ -303,14 +303,14 @@
             valuemask = (r_uint(1) << self.bitsize) - 1
             value = (value >> self.bitshift) & valuemask
             if value_fits_long:
-                return space.wrap(intmask(value))
+                return space.newint(intmask(value))
             else:
-                return space.wrap(value)    # uint => wrapped long object
+                return space.newint(value)    # uint => wrapped long object
         else:
             value = misc.read_raw_unsigned_data(cdata, ctype.size)
             valuemask = (r_ulonglong(1) << self.bitsize) - 1
             value = (value >> self.bitshift) & valuemask
-            return space.wrap(value)      # ulonglong => wrapped long object
+            return space.newint(value)      # ulonglong => wrapped long object
 
     def convert_bitfield_from_object(self, cdata, w_ob):
         ctype = self.ctype
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -38,7 +38,7 @@
     compiler = space.createcompiler()
     pycode = compiler.compile(code, "<init code for '%s'>" % name, 'exec', 0)
     w_globals = space.newdict(module=True)
-    space.setitem_str(w_globals, "__builtins__", space.wrap(space.builtin))
+    space.setitem_str(w_globals, "__builtins__", space.builtin)
     pycode.exec_code(space, w_globals, w_globals)
 
 
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -46,17 +46,17 @@
                         "ctype '%s' is of unknown size", w_obj.name)
     else:
         raise oefmt(space.w_TypeError, "expected a 'cdata' or 'ctype' object")
-    return space.wrap(size)
+    return space.newint(size)
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
 def alignof(space, w_ctype):
     align = w_ctype.alignof()
-    return space.wrap(align)
+    return space.newint(align)
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, following=int)
 def typeoffsetof(space, w_ctype, w_field_or_index, following=0):
     ctype, offset = w_ctype.direct_typeoffsetof(w_field_or_index, following)
-    return space.newtuple([space.wrap(ctype), space.wrap(offset)])
+    return space.newtuple([space.wrap(ctype), space.newint(offset)])
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_cdata=cdataobj.W_CData, offset=int)
 def rawaddressof(space, w_ctype, w_cdata, offset):
@@ -68,7 +68,7 @@
 def getcname(space, w_ctype, replace_with):
     p = w_ctype.name_position
     s = '%s%s%s' % (w_ctype.name[:p], replace_with, w_ctype.name[p:])
-    return space.wrap(s)
+    return space.newtext(s)
 
 # ____________________________________________________________
 
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -26,7 +26,7 @@
         self.libname = libname    # some string that gives the name of the lib
 
     def descr_repr(self):
-        return self.space.wrap("<Lib object for '%s'>" % self.libname)
+        return self.space.newtext("<Lib object for '%s'>" % self.libname)
 
     def make_includes_from(self, c_includes):
         space = self.space
@@ -35,7 +35,7 @@
         while c_includes[num]:
             include_name = rffi.charp2str(c_includes[num])
             try:
-                w_lib1 = space.appexec([space.wrap(include_name)], """(modname):
+                w_lib1 = space.appexec([space.newtext(include_name)], """(modname):
                     mod = __import__(modname, None, None, ['ffi', 'lib'])
                     return mod.lib""")
                 lib1 = space.interp_w(W_LibObject, w_lib1)
@@ -185,7 +185,7 @@
         return w_result
 
     def _get_attr(self, w_attr, is_getattr=False):
-        attr = self.space.str_w(w_attr)
+        attr = self.space.text_w(w_attr)
         try:
             w_value = self._get_attr_elidable(attr)
         except KeyError:
@@ -202,7 +202,7 @@
                     from pypy.interpreter.module import Module
                     return self.space.gettypeobject(Module.typedef)
                 if is_getattr and attr == '__name__':
-                    return self.space.wrap("%s.lib" % self.libname)
+                    return self.space.newtext("%s.lib" % self.libname)
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -36,7 +36,7 @@
 
     def repr(self):
         space = self.space
-        return space.wrap("<clibrary '%s'>" % self.name)
+        return space.newtext("<clibrary '%s'>" % self.name)
 
     @unwrap_spec(w_ctype=W_CType, name=str)
     def load_function(self, w_ctype, name):
@@ -94,4 +94,4 @@
 @unwrap_spec(filename="str_or_None", flags=int)
 def load_library(space, filename, flags=0):
     lib = W_Library(space, filename, flags)
-    return space.wrap(lib)
+    return lib
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -147,7 +147,7 @@
     try:
         return bigint.tolonglong()
     except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+        raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
 
 def as_long(space, w_ob):
     # Same as as_long_long(), but returning an int instead.
@@ -164,7 +164,7 @@
     try:
         return bigint.toint()
     except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+        raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
 
 def as_unsigned_long_long(space, w_ob, strict):
     # (possibly) convert and cast a Python object to an unsigned long long.
@@ -179,7 +179,7 @@
             raise
     else:
         if strict and value < 0:
-            raise OperationError(space.w_OverflowError, space.wrap(neg_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(neg_msg))
         return r_ulonglong(value)
     try:
         bigint = space.bigint_w(w_ob, allow_conversion=False)
@@ -193,9 +193,9 @@
         try:
             return bigint.toulonglong()
         except ValueError:
-            raise OperationError(space.w_OverflowError, space.wrap(neg_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(neg_msg))
         except OverflowError:
-            raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
     else:
         return bigint.ulonglongmask()
 
@@ -213,9 +213,9 @@
         try:
             return bigint.touint()
         except ValueError:
-            raise OperationError(space.w_OverflowError, space.wrap(neg_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(neg_msg))
         except OverflowError:
-            raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
     else:
         return bigint.uintmask()
 
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -167,7 +167,7 @@
     try:
         ctypecls, size, align = PRIMITIVE_TYPES[name]
     except KeyError:
-        raise OperationError(space.w_KeyError, space.wrap(name))
+        raise OperationError(space.w_KeyError, space.newtext(name))
     ctype = ctypecls(space, size, name, len(name), align)
     unique_cache.primitives[name] = ctype
     return ctype
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -131,15 +131,15 @@
 
     if neg == 0:     # positive
         if value <= rffi.cast(rffi.ULONGLONG, sys.maxint):
-            return ffi.space.wrap(intmask(value))
+            return ffi.space.newint(intmask(value))
         else:
-            return ffi.space.wrap(value)
+            return ffi.space.newint(value)
     elif neg == 1:   # negative
         value = rffi.cast(rffi.LONGLONG, value)
         if value >= -sys.maxint-1:
-            return ffi.space.wrap(intmask(value))
+            return ffi.space.newint(intmask(value))
         else:
-            return ffi.space.wrap(value)
+            return ffi.space.newint(value)
 
     if neg == 2:
         got = "%d (0x%x)" % (value, value)
@@ -364,7 +364,7 @@
             while p[j] != ',' and p[j] != '\x00':
                 j += 1
             enname = rffi.charpsize2str(p, j)
-            enumerators_w.append(space.wrap(enname))
+            enumerators_w.append(space.newtext(enname))
 
             gindex = parse_c_type.search_in_globals(ffi.ctxobj.ctx, enname)
             assert gindex >= 0
@@ -493,10 +493,10 @@
                                          field_name, "'")
 
         fields_w[i] = space.newtuple([
-            space.wrap(field_name),
+            space.newtext(field_name),
             w_ctf,
-            space.wrap(fbitsize),
-            space.wrap(field_offset)])
+            space.newint(fbitsize),
+            space.newint(field_offset)])
 
     sflags = 0
     c_flags = rffi.getintfield(s, 'c_flags')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1254,20 +1254,6 @@
     orig_getline = linecache.getline
     try:
         linecache.getline = lambda *args: 'LINE'    # hack: speed up PyPy tests
-        sys.stderr = cStringIO.StringIO()
-        assert f(100) == 300
-        assert sys.stderr.getvalue() == ''
-        assert f(10000) == -42
-        assert matches(sys.stderr.getvalue(), """\
-From cffi callback <function$Zcb1 at 0x$>:
-Traceback (most recent call last):
-  File "$", line $, in Zcb1
-    $
-  File "$", line $, in check_value
-    $
-ValueError: 42
-""")
-        sys.stderr = cStringIO.StringIO()
         bigvalue = 20000
         assert f(bigvalue) == -42
         assert matches(sys.stderr.getvalue(), """\
@@ -1275,59 +1261,6 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
-        sys.stderr = cStringIO.StringIO()
-        bigvalue = 20000
-        assert len(seen) == 0
-        assert ff(bigvalue) == -42
-        assert sys.stderr.getvalue() == ""
-        assert len(seen) == 1
-        exc, val, tb = seen[0]
-        assert exc is OverflowError
-        assert str(val) == "integer 60000 does not fit 'short'"
-        #
-        sys.stderr = cStringIO.StringIO()
-        bigvalue = 20000
-        del seen[:]
-        oops_result = 81
-        assert ff(bigvalue) == 81
-        oops_result = None
-        assert sys.stderr.getvalue() == ""
-        assert len(seen) == 1
-        exc, val, tb = seen[0]
-        assert exc is OverflowError
-        assert str(val) == "integer 60000 does not fit 'short'"
-        #
-        sys.stderr = cStringIO.StringIO()
-        bigvalue = 20000
-        del seen[:]
-        oops_result = "xy"     # not None and not an int!
-        assert ff(bigvalue) == -42
-        oops_result = None
-        assert matches(sys.stderr.getvalue(), """\
-From cffi callback <function$Zcb1 at 0x$>:
-Trying to convert the result back to C:
-OverflowError: integer 60000 does not fit 'short'
-
-During the call to 'onerror', another exception occurred:
-
-TypeError: $integer$
-""")
-        #
-        sys.stderr = cStringIO.StringIO()
-        seen = "not a list"    # this makes the oops() function crash
-        assert ff(bigvalue) == -42
-        assert matches(sys.stderr.getvalue(), """\
-From cffi callback <function$Zcb1 at 0x$>:
-Trying to convert the result back to C:
-OverflowError: integer 60000 does not fit 'short'
-
-During the call to 'onerror', another exception occurred:
-
-Traceback (most recent call last):
-  File "$", line $, in oops
-    $
-AttributeError: 'str' object has no attribute 'append'
-""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -93,12 +93,12 @@
 
     def descr_repr(self, space):
         doc = self.rawfunctype.repr_fn_type(self.ffi, self.fnname)
-        return space.wrap("<FFIFunctionWrapper '%s'>" % (doc,))
+        return space.newtext("<FFIFunctionWrapper '%s'>" % (doc,))
 
     def descr_get_doc(self, space):
         doc = self.rawfunctype.repr_fn_type(self.ffi, self.fnname)
         doc = '%s;\n\nCFFI C function from %s.lib' % (doc, self.modulename)
-        return space.wrap(doc)
+        return space.newtext(doc)
 
     def descr_get(self, space, w_obj, w_type=None):
         # never bind anything, but a __get__ is still present so that
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -743,7 +743,7 @@
     res, consumed, byteorder = runicode.str_decode_utf_16_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
     return space.newtuple([space.newunicode(res), space.newint(consumed),
-                           space.newtext(byteorder)])
+                           space.newint(byteorder)])
 
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
@@ -762,7 +762,7 @@
     res, consumed, byteorder = runicode.str_decode_utf_32_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
     return space.newtuple([space.newunicode(res), space.newint(consumed),
-                           space.newtext(byteorder)])
+                           space.newint(byteorder)])
 
 # ____________________________________________________________
 # Charmap
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -22,7 +22,7 @@
     ]
 
 def _fetch(space, w_dialect, name):
-    return space.findattr(w_dialect, space.wrap(name))
+    return space.findattr(w_dialect, space.newtext(name))
 
 def _get_bool(space, w_src, default):
     if w_src is None:
@@ -134,7 +134,7 @@
                              w_escapechar, w_lineterminator, w_quotechar,
                              w_quoting, w_skipinitialspace, w_strict)
     if space.is_w(w_subtype, space.gettypeobject(W_Dialect.typedef)):
-        return space.wrap(dialect)
+        return dialect
     else:
         subdialect = space.allocate_instance(W_Dialect, w_subtype)
         subdialect.delimiter        = dialect.delimiter
@@ -145,18 +145,18 @@
         subdialect.quoting          = dialect.quoting
         subdialect.skipinitialspace = dialect.skipinitialspace
         subdialect.strict           = dialect.strict
-        return space.wrap(subdialect)
+        return subdialect
 
 
 def _get_escapechar(space, dialect):
     if dialect.escapechar == u'\0':
         return space.w_None
-    return space.wrap(dialect.escapechar)
+    return space.newtext(dialect.escapechar)
 
 def _get_quotechar(space, dialect):
     if dialect.quotechar == u'\0':
         return space.w_None
-    return space.wrap(dialect.quotechar)
+    return space.newtext(dialect.quotechar)
 
 
 W_Dialect.typedef = TypeDef(
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -13,7 +13,7 @@
     def __init__(self, space, dialect, w_fileobj):
         self.space = space
         self.dialect = dialect
-        self.w_filewrite = space.getattr(w_fileobj, space.wrap('write'))
+        self.w_filewrite = space.getattr(w_fileobj, space.newtext('write'))
         # precompute this
         special = dialect.delimiter + dialect.lineterminator
         if dialect.escapechar != '\0':
@@ -26,8 +26,8 @@
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
-        w_error = space.getattr(w_module, space.wrap('Error'))
-        raise OperationError(w_error, space.wrap(msg))
+        w_error = space.getattr(w_module, space.newtext('Error'))
+        raise OperationError(w_error, space.newtext(msg))
 
     def writerow(self, w_fields):
         """Construct and write a CSV record from a sequence of fields.
@@ -115,7 +115,7 @@
         rec.append(dialect.lineterminator)
 
         line = rec.build()
-        return space.call_function(self.w_filewrite, space.wrap(line))
+        return space.call_function(self.w_filewrite, space.newtext(line))
 
     def writerows(self, w_seqseq):
         """Construct and write a series of sequences to a csv file.
diff --git a/pypy/module/_demo/demo.py b/pypy/module/_demo/demo.py
--- a/pypy/module/_demo/demo.py
+++ b/pypy/module/_demo/demo.py
@@ -15,7 +15,7 @@
 
 def get(space, name):
     w_module = space.getbuiltinmodule('_demo')
-    return space.getattr(w_module, space.wrap(name))
+    return space.getattr(w_module, space.newtext(name))
 
 
 @unwrap_spec(repetitions=int)
@@ -27,7 +27,7 @@
     for i in range(repetitions):
         space.call_function(w_callable)
     endtime = time(0)
-    return space.wrap(endtime - starttime)
+    return space.newint(endtime - starttime)
 
 @unwrap_spec(n=int)
 def sieve(space, n):
@@ -55,10 +55,10 @@
     def multiply(self, w_y):
         space = self.space
         y = space.int_w(w_y)
-        return space.wrap(self.x * y)
+        return space.newint(self.x * y)
 
     def fget_x(self, space):
-        return space.wrap(self.x)
+        return space.newint(self.x)
 
     def fset_x(self, space, w_value):
         self.x = space.int_w(w_value)
@@ -66,8 +66,8 @@
 @unwrap_spec(x=int)
 def mytype_new(space, w_subtype, x):
     if x == 3:
-        return space.wrap(MySubType(space, x))
-    return space.wrap(W_MyType(space, x))
+        return MySubType(space, x)
+    return W_MyType(space, x)
 
 getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType)
 
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -27,9 +27,9 @@
     # that we're getting a reasonable value at this point.
     w_value = space.call_function(
         space.w_BlockingIOError,
-        space.wrap(rposix.get_saved_errno()),
-        space.wrap("write could not complete without blocking"),
-        space.wrap(written))
+        space.newint(rposix.get_saved_errno()),
+        space.newtext("write could not complete without blocking"),
+        space.newint(written))
     return OperationError(space.w_BlockingIOError, w_value)
 
 
@@ -115,7 +115,7 @@
     def _readinto(self, space, w_buffer, methodname):
         rwbuffer = space.getarg_w('w*', w_buffer)
         length = rwbuffer.getlength()
-        w_data = space.call_method(self, methodname, space.wrap(length))
+        w_data = space.call_method(self, methodname, space.newint(length))
 
         if not space.isinstance_w(w_data, space.w_str):
             raise oefmt(space.w_TypeError, "%s() should return bytes",
@@ -127,7 +127,7 @@
                         "%d bytes requested, %d returned",
                         methodname, length, len(data))
         rwbuffer.setslice(0, data)
-        return space.wrap(len(data))
+        return space.newint(len(data))
 
 W_BufferedIOBase.typedef = TypeDef(
     '_io._BufferedIOBase', W_IOBase.typedef,
@@ -244,15 +244,15 @@
 
     def closed_get_w(self, space):
         self._check_init(space)
-        return space.getattr(self.w_raw, space.wrap("closed"))
+        return space.getattr(self.w_raw, space.newtext("closed"))
 
     def name_get_w(self, space):
         self._check_init(space)
-        return space.getattr(self.w_raw, space.wrap("name"))
+        return space.getattr(self.w_raw, space.newtext("name"))
 
     def mode_get_w(self, space):
         self._check_init(space)
-        return space.getattr(self.w_raw, space.wrap("mode"))
+        return space.getattr(self.w_raw, space.newtext("mode"))
 
     def readable_w(self, space):
         self._check_init(space)
@@ -271,16 +271,16 @@
         return space.call_method(self.w_raw, "isatty")
 
     def repr_w(self, space):
-        typename = space.type(self).name.decode('utf-8')
+        typename = space.type(self).name
         try:
-            w_name = space.getattr(self, space.wrap("name"))
+            w_name = space.getattr(self, space.newtext("name"))
         except OperationError as e:
             if not e.match(space, space.w_Exception):
                 raise
-            return space.wrap(u"<%s>" % (typename,))
+            return space.newtext("<%s>" % (typename,))
         else:
-            name_repr = space.unicode_w(space.repr(w_name))
-            return space.wrap(u"<%s name=%s>" % (typename, name_repr))
+            name_repr = space.text_w(space.repr(w_name))
+            return space.newtext("<%s name=%s>" % (typename, name_repr))
 
     # ______________________________________________
 
@@ -301,7 +301,7 @@
     def tell_w(self, space):
         self._check_init(space)
         pos = self._raw_tell(space) - self._raw_offset()
-        return space.wrap(pos)
+        return space.newint(pos)
 
     @unwrap_spec(pos=r_longlong, whence=int)
     def seek_w(self, space, pos, whence=0):
@@ -327,7 +327,7 @@
                     newpos = self.pos + offset
                     assert newpos >= 0
                     self.pos = newpos
-                    return space.wrap(current - available + offset)
+                    return space.newint(current - available + offset)
 
         # Fallback: invoke raw seek() method and clear buffer
         with self.lock:
@@ -341,11 +341,11 @@
             self.raw_pos = -1
             if self.readable:
                 self._reader_reset_buf()
-            return space.wrap(n)
+            return space.newint(n)
 
     def _raw_seek(self, space, pos, whence):
         w_pos = space.call_method(self.w_raw, "seek",
-                                  space.wrap(pos), space.wrap(whence))
+                                  space.newint(pos), space.newint(whence))
         pos = space.r_longlong_w(w_pos)
         if pos < 0:
             raise oefmt(space.w_IOError,
@@ -354,7 +354,7 @@
         return pos
 
     def _closed(self, space):
-        return space.is_true(space.getattr(self.w_raw, space.wrap("closed")))
+        return space.is_true(space.getattr(self.w_raw, space.newtext("closed")))
 
     def close_w(self, space):
         self._check_init(space)
@@ -781,7 +781,7 @@
                 self._adjust_position(self.pos + size)
                 if self.pos > self.write_end:
                     self.write_end = self.pos
-                return space.wrap(size)
+                return space.newint(size)
 
             # First write the current buffer
             try:
@@ -809,7 +809,7 @@
                         self.buffer[self.write_end + i] = data[i]
                     self.write_end += size
                     self.pos += size
-                    return space.wrap(size)
+                    return space.newint(size)
                 # Buffer as much as possible
                 for i in range(available):
                     self.buffer[self.write_end + i] = data[i]
@@ -865,7 +865,7 @@
             self.write_end = remaining
             self._adjust_position(remaining)
             self.raw_pos = 0
-        return space.wrap(written)
+        return space.newint(written)
 
     def flush_w(self, space):
         self._check_closed(space, "flush of closed file")
@@ -976,13 +976,13 @@
             if self.w_writer is None:
                 raise oefmt(space.w_ValueError,
                             "I/O operation on uninitialized object")
-            w_meth = space.getattr(self.w_writer, space.wrap(method))
+            w_meth = space.getattr(self.w_writer, space.newtext(method))
             w_result = space.call_args(w_meth, __args__)
         if reader:
             if self.w_reader is None:
                 raise oefmt(space.w_ValueError,
                             "I/O operation on uninitialized object")
-            w_meth = space.getattr(self.w_reader, space.wrap(method))
+            w_meth = space.getattr(self.w_reader, space.newtext(method))
             w_result = space.call_args(w_meth, __args__)
         return w_result
     return method_w
@@ -1019,7 +1019,7 @@
         if self.w_writer is None:
             raise oefmt(space.w_ValueError,
                         "I/O operation on uninitialized object")
-        w_meth = space.getattr(self.w_writer, space.wrap("close"))
+        w_meth = space.getattr(self.w_writer, space.newtext("close"))
         try:
             space.call_args(w_meth, __args__)
         except OperationError as e:
@@ -1030,7 +1030,7 @@
         if self.w_reader is None:
             raise oefmt(space.w_ValueError,
                         "I/O operation on uninitialized object")
-        w_meth = space.getattr(self.w_reader, space.wrap("close"))
+        w_meth = space.getattr(self.w_reader, space.newtext("close"))
         space.call_args(w_meth, __args__)
 
         if e:
@@ -1046,7 +1046,7 @@
         return space.call_method(self.w_reader, "isatty")
 
     def closed_get_w(self, space):
-        return space.getattr(self.w_writer, space.wrap("closed"))
+        return space.getattr(self.w_writer, space.newtext("closed"))
 
 methods = dict((method, interp2app(getattr(W_BufferedRWPair, method + '_w')))
                for method in ['read', 'peek', 'read1', 'readinto', 'readable',
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -71,7 +71,7 @@
         if self.is_closed():
             if message is None:
                 message = "I/O operation on closed file"
-            raise OperationError(space.w_ValueError, space.wrap(message))
+            raise OperationError(space.w_ValueError, space.newtext(message))
 
     def read_w(self, space, w_size=None):
         self._check_closed(space)
@@ -93,16 +93,16 @@
 
         output = self.read(size)
         rwbuffer.setslice(0, output)
-        return space.wrap(len(output))
+        return space.newint(len(output))
 
     def write_w(self, space, w_data):
         self._check_closed(space)
         buf = space.buffer_w(w_data, space.BUF_CONTIG_RO).as_str()
         length = len(buf)
         if length <= 0:
-            return space.wrap(0)
+            return space.newint(0)
         self.write(buf)
-        return space.wrap(length)
+        return space.newint(length)
 
     def truncate_w(self, space, w_size=None):
         self._check_closed(space)
@@ -121,7 +121,7 @@
             self.seek(0, 2)
         else:
             self.seek(pos)
-        return space.wrap(size)
+        return space.newint(size)
 
     def getbuffer_w(self, space):
         self._check_closed(space)
@@ -133,7 +133,7 @@
 
     def tell_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.tell())
+        return space.newint(self.tell())
 
     @unwrap_spec(pos=r_longlong, whence=int)
     def seek_w(self, space, pos, whence=0):
@@ -153,7 +153,7 @@
                         "whence must be between 0 and 2, not %d", whence)
 
         self.seek(pos, whence)
-        return space.wrap(self.tell())
+        return space.newint(self.tell())
 
     def readable_w(self, space):
         self._check_closed(space)
@@ -175,13 +175,13 @@
         return type(self) is not W_BytesIO
 
     def closed_get_w(self, space):
-        return space.wrap(self.is_closed())
+        return space.newbool(self.is_closed())
 
     def getstate_w(self, space):
         self._check_closed(space)
         return space.newtuple([
             space.newbytes(self.getvalue()),
-            space.wrap(self.tell()),
+            space.newint(self.tell()),
             self.getdict(space)])
 
     def setstate_w(self, space, w_state):
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -18,7 +18,7 @@
     def fget(space, obj):
         w_value = getattr(obj, name)
         if w_value is None:
-            raise OperationError(space.w_AttributeError, space.wrap(name))
+            raise OperationError(space.w_AttributeError, space.newtext(name))
         else:
             return w_value
     def fset(space, obj, w_value):
@@ -26,7 +26,7 @@
     def fdel(space, obj):
         w_value = getattr(obj, name)
         if w_value is None:
-            raise OperationError(space.w_AttributeError, space.wrap(name))
+            raise OperationError(space.w_AttributeError, space.newtext(name))
         setattr(obj, name, None)
 
     return GetSetProperty(fget, fset, fdel, cls=cls, doc=doc)
@@ -141,7 +141,7 @@
     def descr_new(space, w_subtype, __args__):
         self = space.allocate_instance(W_FileIO, w_subtype)
         W_FileIO.__init__(self, space)
-        return space.wrap(self)
+        return self
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
@@ -199,7 +199,7 @@
                                                 eintr_retry=False)
                 else:
                     w_fd = space.call_function(w_opener, w_name,
-                                               space.wrap(flags))
+                                               space.newint(flags))
                     try:
                         self.fd = space.int_w(w_fd)
                         if self.fd < 0:
@@ -235,7 +235,7 @@
             if HAS_BLKSIZE and st.st_blksize > 1:
                 self.blksize = st.st_blksize
 
-            space.setattr(self, space.wrap("name"), w_name)
+            space.setattr(self, space.newtext("name"), w_name)
 
             if self.appending:
                 # For consistent behaviour, we explicitly seek to the end of file
@@ -270,10 +270,10 @@
             return 'wb'
 
     def descr_get_mode(self, space):
-        return space.wrap(self._mode())
+        return space.newtext(self._mode())
 
     def get_blksize(self, space):
-        return space.wrap(self.blksize)
+        return space.newint(self.blksize)
 
     def _closed(self, space):
         return self.fd < 0
@@ -282,7 +282,7 @@
         if message is None:
             message = "I/O operation on closed file"
         if self.fd < 0:
-            raise OperationError(space.w_ValueError, space.wrap(message))
+            raise OperationError(space.w_ValueError, space.newtext(message))
 
     def _check_readable(self, space):
         if not self.readable:
@@ -322,13 +322,13 @@
     def _dealloc_warn_w(self, space, w_source):
         if self.fd >= 0 and self.closefd:
             try:
-                msg = (u"unclosed file %s" %
-                       space.unicode_w(space.repr(w_source)))
-                space.warn(space.wrap(msg), space.w_ResourceWarning)
+                msg = ("unclosed file %s" %
+                       space.text_w(space.repr(w_source)))
+                space.warn(space.newtext(msg), space.w_ResourceWarning)
             except OperationError as e:
                 # Spurious errors can appear at shutdown
                 if e.match(space, space.w_Warning):
-                    e.write_unraisable(space, '', space.wrap(self))
+                    e.write_unraisable(space, '', self)
 
     @unwrap_spec(pos=r_longlong, whence=int)
     def seek_w(self, space, pos, whence=0):
@@ -339,7 +339,7 @@
             raise wrap_oserror(space, e,
                                exception_name='w_IOError',
                                eintr_retry=False)
-        return space.wrap(pos)
+        return space.newint(pos)
 
     def tell_w(self, space):
         self._check_closed(space)
@@ -349,15 +349,15 @@
             raise wrap_oserror(space, e,
                                exception_name='w_IOError',
                                eintr_retry=False)
-        return space.wrap(pos)
+        return space.newint(pos)
 
     def readable_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.readable)
+        return space.newbool(self.readable)
 
     def writable_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.writable)
+        return space.newbool(self.writable)
 
     def seekable_w(self, space):
         self._check_closed(space)
@@ -374,7 +374,7 @@
 
     def fileno_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.fd)
+        return space.newint(self.fd)
 
     def isatty_w(self, space):
         self._check_closed(space)
@@ -383,23 +383,23 @@
         except OSError as e:
             raise wrap_oserror(space, e, exception_name='w_IOError',
                                eintr_retry=False)
-        return space.wrap(res)
+        return space.newbool(res)
 
     def repr_w(self, space):
         if self.fd < 0:
-            return space.wrap("<_io.FileIO [closed]>")
+            return space.newtext("<_io.FileIO [closed]>")
 
         closefd = "True" if self.closefd else "False"
 
         if self.w_name is None:
-            return space.wrap(
+            return space.newtext(
                 "<_io.FileIO fd=%d mode='%s' closefd=%s>" % (
                     self.fd, self._mode(), closefd))
         else:
             w_repr = space.repr(self.w_name)
-            return space.wrap(
+            return space.newtext(
                 "<_io.FileIO name=%s mode='%s' closefd=%s>" % (
-                    space.str_w(w_repr), self._mode(), closefd))
+                    space.text_w(w_repr), self._mode(), closefd))
 
     # ______________________________________________
 
@@ -419,7 +419,7 @@
                              exception_name='w_IOError',
                              eintr_retry=True)
 
-        return space.wrap(n)
+        return space.newint(n)
 
     def read_w(self, space, w_size=None):
         self._check_closed(space)
@@ -458,7 +458,7 @@
                              exception_name='w_IOError',
                              eintr_retry=True)
         rwbuffer.setslice(0, buf)
-        return space.wrap(len(buf))
+        return space.newint(len(buf))
 
     def readall_w(self, space):
         self._check_closed(space)
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -71,7 +71,7 @@
         if writing or appending:
             raise oefmt(space.w_ValueError,
                         "can't use U and writing mode at once")
-        space.warn(space.wrap("'U' mode is deprecated ('r' has the same "
+        space.warn(space.newtext("'U' mode is deprecated ('r' has the same "
                               "effect in Python 3.x)"),
                    space.w_DeprecationWarning)
     if text and binary:
@@ -87,8 +87,8 @@
         raise oefmt(space.w_ValueError,
                     "binary mode doesn't take a newline argument")
     w_raw = space.call_function(
-        space.gettypefor(W_FileIO), w_file, space.wrap(rawmode),
-        space.wrap(bool(closefd)), w_opener)
+        space.gettypefor(W_FileIO), w_file, space.newtext(rawmode),
+        space.newbool(closefd), w_opener)
 
     isatty = space.is_true(space.call_method(w_raw, "isatty"))
     line_buffering = buffering == 1 or (buffering < 0 and isatty)
@@ -96,7 +96,7 @@
         buffering = -1
 
     if buffering < 0:
-        buffering = space.c_int_w(space.getattr(w_raw, space.wrap("_blksize")))
+        buffering = space.c_int_w(space.getattr(w_raw, space.newtext("_blksize")))
 
     if buffering < 0:
         raise oefmt(space.w_ValueError, "invalid buffering size")
@@ -115,17 +115,17 @@
     else:
         raise oefmt(space.w_ValueError, "unknown mode: '%s'", mode)
     w_buffer = space.call_function(
-        space.gettypefor(buffer_cls), w_raw, space.wrap(buffering)
+        space.gettypefor(buffer_cls), w_raw, space.newint(buffering)
     )
     if binary:
         return w_buffer
 
     w_wrapper = space.call_function(space.gettypefor(W_TextIOWrapper),
         w_buffer,
-        space.wrap(encoding),
-        space.wrap(errors),
-        space.wrap(newline),
-        space.wrap(line_buffering)
+        space.newtext_or_none(encoding),
+        space.newtext_or_none(errors),
+        space.newtext_or_none(newline),
+        space.newbool(line_buffering)
     )
-    space.setattr(w_wrapper, space.wrap("mode"), space.wrap(mode))
+    space.setattr(w_wrapper, space.newtext("mode"), space.newtext(mode))
     return w_wrapper
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -23,15 +23,15 @@
         return False
     try:
         w_value = error.get_w_value(space)
-        w_errno = space.getattr(w_value, space.wrap("errno"))
-        return space.eq_w(w_errno, space.wrap(EINTR))
+        w_errno = space.getattr(w_value, space.newtext("errno"))
+        return space.eq_w(w_errno, space.newint(EINTR))
     except OperationError:
         return False
 
 def unsupported(space, message):
     w_exc = space.getattr(space.getbuiltinmodule('_io'),
-                          space.wrap('UnsupportedOperation'))
-    return OperationError(w_exc, space.wrap(message))
+                          space.newtext('UnsupportedOperation'))
+    return OperationError(w_exc, space.newtext(message))
 
 # May be called with any object
 def check_readable_w(space, w_obj):
@@ -69,7 +69,7 @@
     def _closed(self, space):
         # This gets the derived attribute, which is *not* __IOBase_closed
         # in most cases!
-        w_closed = space.findattr(self, space.wrap('closed'))
+        w_closed = space.findattr(self, space.newtext('closed'))
         if w_closed is not None and space.is_true(w_closed):
             return True
         return False
@@ -88,7 +88,7 @@
 
     def descr_del(self):
         space = self.space
-        w_closed = space.findattr(self, space.wrap('closed'))
+        w_closed = space.findattr(self, space.newtext('closed'))
         try:
             # If `closed` doesn't exist or can't be evaluated as bool, then
             # the object is probably in an unusable state, so ignore.
@@ -117,7 +117,7 @@
             message = "I/O operation on closed file"
         if self._closed(space):
             raise OperationError(
-                space.w_ValueError, space.wrap(message))
+                space.w_ValueError, space.newtext(message))
 
     def check_closed_w(self, space):
         self._check_closed(space)
@@ -156,7 +156,7 @@
         self._unsupportedoperation(space, "seek")
 
     def tell_w(self, space):
-        return space.call_method(self, "seek", space.wrap(0), space.wrap(1))
+        return space.call_method(self, "seek", space.newint(0), space.newint(1))
 
     def truncate_w(self, space, w_size=None):
         self._unsupportedoperation(space, "truncate")
@@ -166,14 +166,14 @@
 
     def enter_w(self, space):
         self._check_closed(space)
-        return space.wrap(self)
+        return self
 
     def exit_w(self, space, __args__):
         space.call_method(self, "close")
 
     def iter_w(self, space):
         self._check_closed(space)
-        return space.wrap(self)
+        return self
 
     def next_w(self, space):
         w_line = space.call_method(self, "readline")
@@ -203,7 +203,7 @@
         # For backwards compatibility, a (slowish) readline().
         limit = convert_size(space, w_limit)
 
-        has_peek = space.findattr(self, space.wrap("peek"))
+        has_peek = space.findattr(self, space.newtext("peek"))
 
         builder = StringBuilder()
         size = 0
@@ -213,7 +213,7 @@
 
             if has_peek:
                 try:
-                    w_readahead = space.call_method(self, "peek", space.wrap(1))
+                    w_readahead = space.call_method(self, "peek", space.newint(1))
                 except OperationError as e:
                     if trap_eintr(space, e):
                         continue
@@ -243,7 +243,7 @@
                     nreadahead = n
 
             try:
-                w_read = space.call_method(self, "read", space.wrap(nreadahead))
+                w_read = space.call_method(self, "read", space.newint(nreadahead))
             except OperationError as e:
                 if trap_eintr(space, e):
                     continue
@@ -363,7 +363,7 @@
         while True:
             try:
                 w_data = space.call_method(self, "read",
-                                           space.wrap(DEFAULT_BUFFER_SIZE))
+                                           space.newint(DEFAULT_BUFFER_SIZE))
             except OperationError as e:
                 if trap_eintr(space, e):
                     continue
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -28,11 +28,11 @@
 
         if (newline is not None and newline != u"" and newline != u"\n" and
             newline != u"\r" and newline != u"\r\n"):
-            # Not using oefmt() because I don't know how to ues it
+            # Not using oefmt() because I don't know how to use it
             # with unicode
             raise OperationError(space.w_ValueError,
                 space.mod(
-                    space.wrap("illegal newline value: %s"), space.wrap(newline)
+                    space.newtext("illegal newline value: %s"), w_newline
                 )
             )
         if newline is not None:
@@ -45,7 +45,7 @@
             self.w_decoder = space.call_function(
                 space.gettypefor(W_IncrementalNewlineDecoder),
                 space.w_None,
-                space.wrap(int(self.readtranslate))
+                space.newint(int(self.readtranslate))
             )
 
         if not space.is_none(w_initvalue):
@@ -58,9 +58,9 @@
         if self.readnl is None:
             w_readnl = space.w_None
         else:
-            w_readnl = space.str(space.wrap(self.readnl))
+            w_readnl = space.str(space.newunicode(self.readnl)) # YYY
         return space.newtuple([
-            w_initialval, w_readnl, space.wrap(self.pos), w_dict
+            w_initialval, w_readnl, space.newint(self.pos), w_dict
         ])
 
     def descr_setstate(self, space, w_state):
@@ -105,7 +105,7 @@
         if self.buf is None:
             if message is None:
                 message = "I/O operation on closed file"
-            raise OperationError(space.w_ValueError, space.wrap(message))
+            raise OperationError(space.w_ValueError, space.newtext(message))
 
     def resize_buffer(self, newlength):
         if len(self.buf) > newlength:
@@ -139,7 +139,7 @@
 
         if self.writenl:
             w_decoded = space.call_method(
-                w_decoded, "replace", space.wrap("\n"), space.wrap(self.writenl)
+                w_decoded, "replace", space.newtext("\n"), space.newunicode(self.writenl)
             )
 
         string = space.unicode_w(w_decoded)
@@ -147,7 +147,7 @@
 
         if size:
             self.write(string)
-        return space.wrap(orig_size)
+        return space.newint(orig_size)
 
     def read_w(self, space, w_size=None):
         self._check_closed(space)
@@ -155,21 +155,21 @@
         start = self.pos
         available = len(self.buf) - start
         if available <= 0:
-            return space.wrap(u"")
+            return space.newunicode(u"")
         if size >= 0 and size <= available:
             end = start + size
         else:
             end = len(self.buf)
         assert 0 <= start <= end
         self.pos = end
-        return space.wrap(u''.join(self.buf[start:end]))
+        return space.newunicode(u''.join(self.buf[start:end]))
 
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)
         limit = convert_size(space, w_limit)
 
         if self.pos >= len(self.buf):
-            return space.wrap(u"")
+            return space.newunicode(u"")
 
         start = self.pos
         if limit < 0 or limit > len(self.buf) - self.pos:
@@ -190,7 +190,7 @@
             endpos = end
         assert endpos >= 0
         self.pos = endpos
-        return space.wrap(u"".join(self.buf[start:endpos]))
+        return space.newunicode(u"".join(self.buf[start:endpos]))
 
     @unwrap_spec(pos=int, mode=int)
     def seek_w(self, space, pos, mode=0):
@@ -212,7 +212,7 @@
 
         assert pos >= 0
         self.pos = pos
-        return space.wrap(pos)
+        return space.newint(pos)
 
     def truncate_w(self, space, w_size=None):
         self._check_closed(space)
@@ -227,11 +227,11 @@
         if size < len(self.buf):
             self.resize_buffer(size)
 
-        return space.wrap(size)
+        return space.newint(size)
 
     def getvalue_w(self, space):
         self._check_closed(space)
-        return space.wrap(u''.join(self.buf))
+        return space.newunicode(u''.join(self.buf))
 
     def readable_w(self, space):
         self._check_closed(space)
@@ -253,7 +253,7 @@
         return type(self) is not W_StringIO
 
     def closed_get_w(self, space):
-        return space.wrap(self.buf is None)
+        return space.newbool(self.buf is None)
 
     def line_buffering_get_w(self, space):
         return space.w_False
@@ -261,7 +261,7 @@
     def newlines_get_w(self, space):
         if self.w_decoder is None:
             return space.w_None
-        return space.getattr(self.w_decoder, space.wrap("newlines"))
+        return space.getattr(self.w_decoder, space.newtext("newlines"))
 
 
 W_StringIO.typedef = TypeDef(
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -29,17 +29,17 @@
 
     def __init__(self, space):
         self.w_newlines_dict = {
-            SEEN_CR: space.wrap(u"\r"),
-            SEEN_LF: space.wrap(u"\n"),
-            SEEN_CRLF: space.wrap(u"\r\n"),
+            SEEN_CR: space.newunicode(u"\r"),
+            SEEN_LF: space.newunicode(u"\n"),
+            SEEN_CRLF: space.newunicode(u"\r\n"),
             SEEN_CR | SEEN_LF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\n")]),
+                [space.newunicode(u"\r"), space.newunicode(u"\n")]),
             SEEN_CR | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\r\n")]),
+                [space.newunicode(u"\r"), space.newunicode(u"\r\n")]),
             SEEN_LF | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\n"), space.wrap(u"\r\n")]),
+                [space.newunicode(u"\n"), space.newunicode(u"\r\n")]),
             SEEN_CR | SEEN_LF | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\n"), space.wrap(u"\r\n")]),
+                [space.newunicode(u"\r"), space.newunicode(u"\n"), space.newunicode(u"\r\n")]),
             }
 
     @unwrap_spec(translate=int)
@@ -47,7 +47,7 @@
         self.w_decoder = w_decoder
         self.translate = translate
         if space.is_none(w_errors):
-            self.w_errors = space.wrap("strict")
+            self.w_errors = space.newtext("strict")
         else:
             self.w_errors = w_errors
 
@@ -65,7 +65,7 @@
         # decode input (with the eventual \r from a previous pass)
         if not space.is_w(self.w_decoder, space.w_None):
             w_output = space.call_method(self.w_decoder, "decode",
-                                         w_input, space.wrap(final))
+                                         w_input, space.newbool(final))
         else:
             w_output = w_input
 
@@ -91,7 +91,7 @@
                 output_len -= 1
 
         if output_len == 0:
-            return space.wrap(u"")
+            return space.newunicode(u"")
 
         # Record which newlines are read and do newline translation if
         # desired, all in one pass.
@@ -146,7 +146,7 @@
             output = builder.build()
 
         self.seennl |= seennl
-        return space.wrap(output)
+        return space.newunicode(output)
 
     def reset_w(self, space):
         self.seennl = 0
@@ -165,7 +165,7 @@
         flag <<= 1
         if self.pendingcr:
             flag |= 1
-        return space.newtuple([w_buffer, space.wrap(flag)])
+        return space.newtuple([w_buffer, space.newint(flag)])
 
     def setstate_w(self, space, w_state):
         w_buffer, w_flag = space.unpackiterable(w_state, 2)
@@ -174,7 +174,7 @@
         flag >>= 1
 
         if self.w_decoder and not space.is_w(self.w_decoder, space.w_None):
-            w_state = space.newtuple([w_buffer, space.wrap(flag)])
+            w_state = space.newtuple([w_buffer, space.newint(flag)])
             space.call_method(self.w_decoder, "setstate", w_state)
 
 W_IncrementalNewlineDecoder.typedef = TypeDef(
@@ -272,7 +272,7 @@
 
 def _determine_encoding(space, encoding, w_buffer):
     if encoding is not None:
-        return space.wrap(encoding)
+        return space.newtext(encoding)
 
     # Try os.device_encoding(fileno)
     try:
@@ -283,21 +283,21 @@
                 e.match(space, space.fromcache(Cache).w_unsupportedoperation)):
             raise
     else:
-        w_os = space.call_method(space.builtin, '__import__', space.wrap('os'))
+        w_os = space.call_method(space.builtin, '__import__', space.newtext('os'))
         w_encoding = space.call_method(w_os, 'device_encoding', w_fileno)
         if space.isinstance_w(w_encoding, space.w_unicode):
             return w_encoding
 
     try:
         w_locale = space.call_method(space.builtin, '__import__',
-                                     space.wrap('locale'))
+                                     space.newtext('locale'))
         w_encoding = space.call_method(w_locale, 'getpreferredencoding',
                                        space.w_False)
     except OperationError as e:
         # getpreferredencoding() may also raise ImportError
         if not e.match(space, space.w_ImportError):
             raise
-        return space.wrap('ascii')
+        return space.newtext('ascii')
     else:
         if space.isinstance_w(w_encoding, space.w_unicode):
             return w_encoding
@@ -383,7 +383,7 @@
         self.w_encoding = _determine_encoding(space, encoding, w_buffer)
 
         if space.is_none(w_errors):
-            w_errors = space.wrap("strict")
+            w_errors = space.newtext("strict")
         self.w_errors = w_errors
 
         if space.is_none(w_newline):
@@ -413,9 +413,9 @@
             self.writenl = None
 
         w_codec = interp_codecs.lookup_codec(space,
-                                             space.str_w(self.w_encoding))
+                                             space.text_w(self.w_encoding))
         if not space.is_true(space.getattr(w_codec,
-                                           space.wrap('_is_text_encoding'))):
+                                           space.newtext('_is_text_encoding'))):
             msg = ("%R is not a text encoding; "
                    "use codecs.open() to handle arbitrary codecs")
             raise oefmt(space.w_LookupError, msg, self.w_encoding)
@@ -427,7 +427,7 @@
             if self.readuniversal:
                 self.w_decoder = space.call_function(
                     space.gettypeobject(W_IncrementalNewlineDecoder.typedef),
-                    self.w_decoder, space.wrap(self.readtranslate))
+                    self.w_decoder, space.newbool(self.readtranslate))
 
         # build the encoder object
         if space.is_true(space.call_method(w_buffer, "writable")):
@@ -437,15 +437,15 @@
         self.seekable = space.is_true(space.call_method(w_buffer, "seekable"))
         self.telling = self.seekable
 
-        self.has_read1 = space.findattr(w_buffer, space.wrap("read1"))
+        self.has_read1 = space.findattr(w_buffer, space.newtext("read1"))
 
         self.encoding_start_of_stream = False
         if self.seekable and self.w_encoder:
             self.encoding_start_of_stream = True
             w_cookie = space.call_method(self.w_buffer, "tell")
-            if not space.eq_w(w_cookie, space.wrap(0)):
+            if not space.eq_w(w_cookie, space.newint(0)):
                 self.encoding_start_of_stream = False
-                space.call_method(self.w_encoder, "setstate", space.wrap(0))
+                space.call_method(self.w_encoder, "setstate", space.newint(0))
 
         self.state = STATE_OK
 
@@ -465,10 +465,10 @@
         W_TextIOBase._check_closed(self, space, message)
 
     def __w_attr_repr(self, space, name):
-        w_attr = space.findattr(self, space.wrap(name))
+        w_attr = space.findattr(self, space.newtext(name))
         if w_attr is None:
-            return space.wrap("")
-        return space.mod(space.wrap("%s=%%r " % name), w_attr)
+            return space.newtext("")
+        return space.mod(space.newtext("%s=%%r " % name), w_attr)
 
     def descr_repr(self, space):
         self._check_init(space)
@@ -476,7 +476,7 @@
                                  self.__w_attr_repr(space, 'mode'),
                                  self.w_encoding])
         return space.mod(
-            space.wrap("<_io.TextIOWrapper %s%sencoding=%r>"), w_args
+            space.newtext("<_io.TextIOWrapper %s%sencoding=%r>"), w_args
         )
 
     def readable_w(self, space):
@@ -501,17 +501,17 @@
 
     def closed_get_w(self, space):
         self._check_attached(space)
-        return space.getattr(self.w_buffer, space.wrap("closed"))
+        return space.getattr(self.w_buffer, space.newtext("closed"))
 
     def newlines_get_w(self, space):
         self._check_attached(space)
         if self.w_decoder is None:
             return space.w_None
-        return space.findattr(self.w_decoder, space.wrap("newlines"))
+        return space.findattr(self.w_decoder, space.newtext("newlines"))
 
     def name_get_w(self, space):
         self._check_attached(space)
-        return space.getattr(self.w_buffer, space.wrap("name"))
+        return space.getattr(self.w_buffer, space.newtext("name"))
 
     def flush_w(self, space):
         self._check_attached(space)
@@ -530,7 +530,7 @@
     def close_w(self, space):
         self._check_attached(space)
         if not space.is_true(space.getattr(self.w_buffer,
-                                           space.wrap("closed"))):
+                                           space.newtext("closed"))):
             try:
                 space.call_method(self, "flush")
             finally:
@@ -594,7 +594,7 @@
         # Read a chunk, decode it, and put the result in self._decoded_chars
         w_input = space.call_method(self.w_buffer,
                                     "read1" if self.has_read1 else "read",
-                                    space.wrap(self.chunk_size))
+                                    space.newint(self.chunk_size))
 
         if not space.isinstance_w(w_input, space.w_str):
             msg = "decoder getstate() should have returned a bytes " \
@@ -603,7 +603,7 @@
 
         eof = space.len_w(w_input) == 0
         w_decoded = space.call_method(self.w_decoder, "decode",
-                                      w_input, space.wrap(eof))
+                                      w_input, space.newbool(eof))
         check_decoded(space, w_decoded)
         self._set_decoded_chars(space.unicode_w(w_decoded))
         if space.len_w(w_decoded) > 0:
@@ -641,7 +641,7 @@
             w_bytes = space.call_method(self.w_buffer, "read")
             w_decoded = space.call_method(self.w_decoder, "decode", w_bytes, space.w_True)
             check_decoded(space, w_decoded)
-            w_result = space.wrap(self._get_decoded_chars(-1))
+            w_result = space.newunicode(self._get_decoded_chars(-1))
             w_final = space.add(w_result, w_decoded)
             self.snapshot = None
             return w_final
@@ -667,7 +667,7 @@
                     continue
                 raise
 
-        return space.wrap(builder.build())
+        return space.newunicode(builder.build())
 
     def readline_w(self, space, w_limit=None):
         self._check_attached(space)
@@ -758,9 +758,9 @@
             line = u''.join(chunks)
 
         if line:
-            return space.wrap(line)
+            return space.newunicode(line)
         else:
-            return space.wrap(u'')
+            return space.newunicode(u'')
 
     # _____________________________________________________________
     # write methods
@@ -784,8 +784,8 @@
             if text.find(u'\n') >= 0:
                 haslf = True
         if haslf and self.writetranslate and self.writenl:
-            w_text = space.call_method(w_text, "replace", space.wrap(u'\n'),
-                                       space.wrap(self.writenl))
+            w_text = space.call_method(w_text, "replace", space.newunicode(u'\n'),
+                                       space.newunicode(self.writenl))
             text = space.unicode_w(w_text)
 
         needflush = False
@@ -819,7 +819,7 @@
         if self.w_decoder:
             space.call_method(self.w_decoder, "reset")
 
-        return space.wrap(textlen)
+        return space.newint(textlen)
 
     def _writeflush(self, space):
         if not self.pending_bytes:
@@ -862,14 +862,14 @@
         else:
             space.call_method(self.w_decoder, "setstate",
                               space.newtuple([space.newbytes(""),
-                                              space.wrap(cookie.dec_flags)]))
+                                              space.newint(cookie.dec_flags)]))
 
     def _encoder_setstate(self, space, cookie):
         if cookie.start_pos == 0 and cookie.dec_flags == 0:
             space.call_method(self.w_encoder, "reset")
             self.encoding_start_of_stream = True
         else:
-            space.call_method(self.w_encoder, "setstate", space.wrap(0))
+            space.call_method(self.w_encoder, "setstate", space.newint(0))
             self.encoding_start_of_stream = False
 
     @unwrap_spec(whence=int)
@@ -882,7 +882,7 @@
 
         if whence == 1:
             # seek relative to current position
-            if not space.is_true(space.eq(w_pos, space.wrap(0))):
+            if not space.is_true(space.eq(w_pos, space.newint(0))):
                 self._unsupportedoperation(
                     space, "can't do nonzero cur-relative seeks")
             # Seeking to the current position should attempt to sync the
@@ -891,7 +891,7 @@
 
         elif whence == 2:
             # seek relative to end of file
-            if not space.is_true(space.eq(w_pos, space.wrap(0))):
+            if not space.is_true(space.eq(w_pos, space.newint(0))):
                 self._unsupportedoperation(
                     space, "can't do nonzero end-relative seeks")
             space.call_method(self, "flush")
@@ -900,14 +900,14 @@
             if self.w_decoder:
                 space.call_method(self.w_decoder, "reset")
             return space.call_method(self.w_buffer, "seek",
-                                     w_pos, space.wrap(whence))
+                                     w_pos, space.newint(whence))
 
         elif whence != 0:
             raise oefmt(space.w_ValueError,
                         "invalid whence (%d, should be 0, 1 or 2)",
                         whence)
 
-        if space.is_true(space.lt(w_pos, space.wrap(0))):
+        if space.is_true(space.lt(w_pos, space.newint(0))):
             r = space.str_w(space.repr(w_pos))
             raise oefmt(space.w_ValueError,
                         "negative seek position %s", r)
@@ -919,7 +919,7 @@
         cookie = PositionCookie(space.bigint_w(w_pos))
 
         # Seek back to the safe start point
-        space.call_method(self.w_buffer, "seek", space.wrap(cookie.start_pos))
+        space.call_method(self.w_buffer, "seek", space.newint(cookie.start_pos))
 
         self._set_decoded_chars(None)
         self.snapshot = None
@@ -931,7 +931,7 @@
         if cookie.chars_to_skip:
             # Just like _read_chunk, feed the decoder and save a snapshot.
             w_chunk = space.call_method(self.w_buffer, "read",
-                                        space.wrap(cookie.bytes_to_feed))
+                                        space.newint(cookie.bytes_to_feed))
             if not space.isinstance_w(w_chunk, space.w_str):
                 msg = "underlying read() should have returned " \
                       "a bytes object, not '%T'"
@@ -941,7 +941,7 @@
                                              space.bytes_w(w_chunk))
 
             w_decoded = space.call_method(self.w_decoder, "decode",
-                                          w_chunk, space.wrap(cookie.need_eof))
+                                          w_chunk, space.newbool(cookie.need_eof))
             check_decoded(space, w_decoded)
             self._set_decoded_chars(space.unicode_w(w_decoded))
 
@@ -1034,8 +1034,8 @@
             else:
                 # We didn't get enough decoded data; signal EOF to get more.
                 w_decoded = space.call_method(self.w_decoder, "decode",
-                                              space.wrap(""),
-                                              space.wrap(1)) # final=1
+                                              space.newbytes(""),
+                                              space.newint(1)) # final=1
                 check_decoded(space, w_decoded)
                 chars_decoded += len(space.unicode_w(w_decoded))
                 cookie.need_eof = 1
@@ -1052,7 +1052,7 @@
 
     def chunk_size_get_w(self, space):
         self._check_attached(space)
-        return space.wrap(self.chunk_size)
+        return space.newint(self.chunk_size)
 
     def chunk_size_set_w(self, space, w_size):
         self._check_attached(space)
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -54,7 +54,7 @@
             calls_repr = "None"
         else:
             calls_repr = space.str_w(space.repr(self.w_calls))
-        return space.wrap('("%s", %d, %d, %f, %f, %s)' % (
+        return space.newtext('("%s", %d, %d, %f, %f, %s)' % (
             frame_repr, self.callcount, self.reccallcount,
             self.tt, self.it, calls_repr))
 
@@ -82,7 +82,7 @@
 

From pypy.commits at gmail.com  Tue Feb 14 11:37:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 08:37:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <58a3323c.c6d11c0a.75383.5b0f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90116:b30c111d304e
Date: 2017-02-14 17:36 +0100
http://bitbucket.org/pypy/pypy/changeset/b30c111d304e/

Log:	merge heads

diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -58,7 +58,7 @@
                 errorpos = rffi.cast(lltype.Signed, errorposp[0])
                 if errorpos == -1:
                     raise MemoryError
-                errmsg = _errmsg(u"pypy_wchar2char")
+                errmsg = _errmsg("pypy_wchar2char")
                 errorhandler('strict', 'filesystemencoding', errmsg, u,
                              errorpos, errorpos + 1)
             return rffi.charp2str(sbuf)
@@ -81,7 +81,7 @@
             ubuf = pypy_char2wchar(sbuf, sizep)
         try:
             if not ubuf:
-                errmsg = _errmsg(u"pypy_char2wchar")
+                errmsg = _errmsg("pypy_char2wchar")
                 errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
             size = rffi.cast(lltype.Signed, sizep[0])
             return rawwcharp2unicoden(ubuf, size)
@@ -91,7 +91,7 @@
 
 def _errmsg(what):
     # I *think* that the functions in locale_codec.c don't set errno
-    return u"%s failed" % what
+    return "%s failed" % what
 
 
 class scoped_unicode2rawwcharp:
diff --git a/pypy/module/cpyext/include/methodobject.h b/pypy/module/cpyext/include/methodobject.h
--- a/pypy/module/cpyext/include/methodobject.h
+++ b/pypy/module/cpyext/include/methodobject.h
@@ -7,14 +7,6 @@
 extern "C" {
 #endif
 
-typedef struct
-{
-    PyObject_HEAD
-    PyMethodDef *m_ml; /* Description of the C function to call */
-    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
-    PyObject    *m_module; /* The __module__ attribute, can be anything */
-} PyCFunctionObject;
-
 /* Flag passed to newmethodobject */
 #define METH_VARARGS  0x0001
 #define METH_KEYWORDS 0x0002
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -9,8 +9,8 @@
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
-    METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function,
-    build_type_checkers, cpython_api, cpython_struct, generic_cpy_call,
+    METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
+    build_type_checkers, cpython_api, generic_cpy_call,
     PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
@@ -18,14 +18,7 @@
 PyMethodDef = cts.gettype('PyMethodDef')
 PyCFunction = cts.gettype('PyCFunction')
 PyCFunctionKwArgs = cts.gettype('PyCFunctionWithKeywords')
-PyCFunctionObjectStruct = cpython_struct(
-    'PyCFunctionObject',
-    PyObjectFields + (
-     ('m_ml', lltype.Ptr(PyMethodDef)),
-     ('m_self', PyObject),
-     ('m_module', PyObject),
-     ))
-PyCFunctionObject = lltype.Ptr(PyCFunctionObjectStruct)
+PyCFunctionObject = cts.gettype('PyCFunctionObject*')
 
 @bootstrap_function
 def init_methodobject(space):
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -188,6 +188,14 @@
 };
 typedef struct PyMethodDef PyMethodDef;
 
+typedef struct {
+    PyObject_HEAD
+    PyMethodDef *m_ml; /* Description of the C function to call */
+    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
+    PyObject    *m_module; /* The __module__ attribute, can be anything */
+    PyObject    *m_weakreflist; /* List of weak references */
+} PyCFunctionObject;
+
 /* from structmember.h */
 typedef struct PyMemberDef {
     /* Current version, use this */

From pypy.commits at gmail.com  Tue Feb 14 11:44:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 08:44:14 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix some newbytes/newtext
 mistakes (they show up as conflicts in case
Message-ID: <58a333de.0a081c0a.33804.521d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90117:177cb9a0a3c2
Date: 2017-02-14 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/177cb9a0a3c2/

Log:	fix some newbytes/newtext mistakes (they show up as conflicts in
	case the py3.5 branch already did the replacement choosing the other
	one)

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -124,14 +124,14 @@
 def PyString_FromStringAndSize(space, char_p, length):
     if char_p:
         s = rffi.charpsize2str(char_p, length)
-        return make_ref(space, space.newtext(s))
+        return make_ref(space, space.newbytes(s))
     else:
         return rffi.cast(PyObject, new_empty_str(space, length))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyString_FromString(space, char_p):
     s = rffi.charp2str(char_p)
-    return space.newtext(s)
+    return space.newbytes(s)
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -361,7 +361,7 @@
                 wrapper_func_kwds, doc, func_voidp, offset=offset)
         dict_w[method_name] = w_obj
     if pto.c_tp_doc:
-        dict_w['__doc__'] = space.newbytes(
+        dict_w['__doc__'] = space.newtext(
             rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -432,7 +432,7 @@
         else:
             path = space.str0_w(w_path)
             fullpath = rposix.getfullpathname(path)
-            w_fullpath = space.newtext(fullpath)
+            w_fullpath = space.newbytes(fullpath)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     else:

From pypy.commits at gmail.com  Tue Feb 14 11:44:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 08:44:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 111d8e1779ff
Message-ID: <58a333e0.dea2df0a.e85df.7b83@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90118:b39d70ae1e8e
Date: 2017-02-14 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/b39d70ae1e8e/

Log:	hg merge 111d8e1779ff

diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py
--- a/pypy/module/cpyext/bytearrayobject.py
+++ b/pypy/module/cpyext/bytearrayobject.py
@@ -83,7 +83,7 @@
             space.call_method(w_obj, 'extend', space.newbytes('\x00' * (newlen - oldlen)))
         elif oldlen > newlen:
             assert newlen >= 0
-            space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen))
+            space.delslice(w_obj, space.newint(newlen), space.newint(oldlen))
         return 0
     else:
         raise oefmt(space.w_TypeError,
diff --git a/pypy/module/cpyext/mapping.py b/pypy/module/cpyext/mapping.py
--- a/pypy/module/cpyext/mapping.py
+++ b/pypy/module/cpyext/mapping.py
@@ -44,14 +44,14 @@
 def PyMapping_GetItemString(space, w_obj, key):
     """Return element of o corresponding to the object key or NULL on failure.
     This is the equivalent of the Python expression o[key]."""
-    w_key = space.wrap(rffi.charp2str(key))
+    w_key = space.newtext(rffi.charp2str(key))
     return space.getitem(w_obj, w_key)
 
 @cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1)
 def PyMapping_SetItemString(space, w_obj, key, w_value):
     """Map the object key to the value v in object o. Returns -1 on failure.
     This is the equivalent of the Python statement o[key] = v."""
-    w_key = space.wrap(rffi.charp2str(key))
+    w_key = space.newtext(rffi.charp2str(key))
     space.setitem(w_obj, w_key, w_value)
     return 0
 
@@ -72,7 +72,7 @@
     This is equivalent to o[key], returning True on success and False
     on an exception.  This function always succeeds."""
     try:
-        w_key = space.wrap(rffi.charp2str(key))
+        w_key = space.newtext(rffi.charp2str(key))
         space.getitem(w_obj, w_key)
         return 1
     except:
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -35,9 +35,9 @@
     if traceback.next is None:
         w_next_traceback = None
     else:
-        w_next_traceback = space.wrap(traceback.next)
+        w_next_traceback = traceback.next
     py_traceback.c_tb_next = rffi.cast(PyTracebackObject, make_ref(space, w_next_traceback))
-    py_traceback.c_tb_frame = rffi.cast(PyFrameObject, make_ref(space, space.wrap(traceback.frame)))
+    py_traceback.c_tb_frame = rffi.cast(PyFrameObject, make_ref(space, traceback.frame))
     rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti)
     rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno())
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -61,7 +61,7 @@
                                 tag="cpyext_1")
 
 def PyDescr_NewGetSet(space, getset, w_type):
-    return space.wrap(W_GetSetPropertyEx(getset, w_type))
+    return W_GetSetPropertyEx(getset, w_type)
 
 def make_GetSet(space, getsetprop):
     py_getsetdef = lltype.malloc(PyGetSetDef, flavor='raw')
@@ -244,7 +244,7 @@
             if not name:
                 break
             name = rffi.charp2str(name)
-            w_descr = space.wrap(W_MemberDescr(member, w_type))
+            w_descr = W_MemberDescr(member, w_type)
             dict_w[name] = w_descr
             i += 1
 
@@ -356,9 +356,9 @@
             continue
         w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func,
                 wrapper_func_kwds, doc, func_voidp, offset=offset)
-        dict_w[method_name] = space.wrap(w_obj)
+        dict_w[method_name] = w_obj
     if pto.c_tp_doc:
-        dict_w['__doc__'] = space.newbytes(
+        dict_w['__doc__'] = space.newtext(
             rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
@@ -502,7 +502,7 @@
         elif pto.c_tp_as_mapping and pto.c_tp_as_mapping.c_mp_subscript:
             self.flag_map_or_seq = 'M'
         if pto.c_tp_doc:
-            self.w_doc = space.newbytes(
+            self.w_doc = space.newtext(
                 rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
 
 @bootstrap_function
@@ -641,7 +641,7 @@
             # point we might get into troubles by doing make_ref() when
             # things are not initialized yet.  So in this case, simply use
             # str2charp() and "leak" the string.
-        w_typename = space.getattr(w_type, space.wrap('__name__'))
+        w_typename = space.getattr(w_type, space.newtext('__name__'))
         heaptype = cts.cast('PyHeapTypeObject*', pto)
         heaptype.c_ht_name = make_ref(space, w_typename)
         from pypy.module.cpyext.unicodeobject import PyUnicode_AsUTF8
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -416,7 +416,7 @@
     is NULL."""
     if wchar_p:
         s = rffi.wcharpsize2unicode(wchar_p, length)
-        return make_ref(space, space.wrap(s))
+        return make_ref(space, space.newunicode(s))
     else:
         return new_empty_unicode(space, length)
 
@@ -442,7 +442,7 @@
         # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
         encoding = PyUnicode_GetDefaultEncoding(space)
     w_str = space.newbytes(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
+    w_encoding = space.newtext(rffi.charp2str(encoding))
     if errors:
         w_errors = space.newbytes(rffi.charp2str(errors))
     else:
@@ -848,7 +848,7 @@
             if decimal >= 0:
                 ch = unichr(ord('0') + decimal)
         result.append(ch)
-    return space.wrap(result.build())
+    return space.newunicode(result.build())
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-2)
 def PyUnicode_Compare(space, w_left, w_right):

From pypy.commits at gmail.com  Tue Feb 14 11:47:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 08:47:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge d1caa90c2615
Message-ID: <58a33487.d3471c0a.ed904.a722@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90119:ffee6934a103
Date: 2017-02-14 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/ffee6934a103/

Log:	hg merge d1caa90c2615

diff --git a/pypy/module/_jitlog/interp_jitlog.py b/pypy/module/_jitlog/interp_jitlog.py
--- a/pypy/module/_jitlog/interp_jitlog.py
+++ b/pypy/module/_jitlog/interp_jitlog.py
@@ -12,7 +12,7 @@
 
 def JitlogError(space, e):
     w_JitlogError = space.fromcache(Cache).w_JitlogError
-    return OperationError(w_JitlogError, space.wrap(e.msg))
+    return OperationError(w_JitlogError, space.newtext(e.msg))
 
 @unwrap_spec(fileno=int)
 def enable(space, fileno):
diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py
--- a/pypy/module/_minimal_curses/interp_curses.py
+++ b/pypy/module/_minimal_curses/interp_curses.py
@@ -22,8 +22,8 @@
 def convert_error(space, error):
     msg = error.msg
     w_module = space.getbuiltinmodule('_minimal_curses')
-    w_exception_class = space.getattr(w_module, space.wrap('error'))
-    w_exception = space.call_function(w_exception_class, space.wrap(msg))
+    w_exception_class = space.getattr(w_module, space.newtext('error'))
+    w_exception = space.call_function(w_exception_class, space.newtext(msg))
     return OperationError(w_exception_class, w_exception)
 
 def _curses_setupterm_null(fd):
@@ -44,9 +44,9 @@
 def setupterm(space, w_termname=None, fd=-1):
     if fd == -1:
         w_stdout = space.getattr(space.getbuiltinmodule('sys'),
-                                 space.wrap('stdout'))
+                                 space.newtext('stdout'))
         fd = space.int_w(space.call_function(space.getattr(w_stdout,
-                                             space.wrap('fileno'))))
+                                             space.newtext('fileno'))))
     try:
         if space.is_none(w_termname):
             _curses_setupterm_null(fd)
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -47,7 +47,7 @@
 
 def VMProfError(space, e):
     w_VMProfError = space.fromcache(Cache).w_VMProfError
-    return OperationError(w_VMProfError, space.wrap(e.msg))
+    return OperationError(w_VMProfError, space.newtext(e.msg))
 
 
 @unwrap_spec(fileno=int, period=float)
@@ -60,9 +60,9 @@
     Must be smaller than 1.0
     """
     w_modules = space.sys.get('modules')
-    #if space.contains_w(w_modules, space.wrap('_continuation')):
-    #    space.warn(space.wrap("Using _continuation/greenlet/stacklet together "
-    #                          "with vmprof will crash"),
+    #if space.contains_w(w_modules, space.newtext('_continuation')):
+    #    space.warn(space.newtext("Using _continuation/greenlet/stacklet together "
+    #                             "with vmprof will crash"),
     #               space.w_RuntimeWarning)
     try:
         rvmprof.enable(fileno, period)
diff --git a/pypy/module/pypyjit/hooks.py b/pypy/module/pypyjit/hooks.py
--- a/pypy/module/pypyjit/hooks.py
+++ b/pypy/module/pypyjit/hooks.py
@@ -18,9 +18,9 @@
             try:
                 try:
                     space.call_function(cache.w_abort_hook,
-                        space.wrap(jitdriver.name),
+                        space.newtext(jitdriver.name),
                         wrap_greenkey(space, jitdriver, greenkey, greenkey_repr),
-                        space.wrap(Counters.counter_names[reason]),
+                        space.newtext(Counters.counter_names[reason]),
                         space.newlist(oplist_w)
                     )
                 except OperationError as e:
@@ -38,7 +38,7 @@
             try:
                 try:
                     space.call_function(cache.w_trace_too_long_hook,
-                        space.wrap(jitdriver.name),
+                        space.newtext(jitdriver.name),
                         wrap_greenkey(space, jitdriver, greenkey, greenkey_repr))
                 except OperationError as e:
                     e.write_unraisable(space, "jit hook", cache.w_trace_too_long_hook)
@@ -69,7 +69,7 @@
             try:
                 try:
                     space.call_function(cache.w_compile_hook,
-                                        space.wrap(w_debug_info))
+                                        w_debug_info)
                 except OperationError as e:
                     e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
             finally:
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -361,19 +361,19 @@
     w_times = space.newdict()
     if ll_times:
         for i in range(len(ll_times)):
-            w_key = space.newtuple([space.wrap(ll_times[i].type),
-                                    space.wrap(ll_times[i].number)])
+            w_key = space.newtuple([space.newtext(ll_times[i].type),
+                                    space.newint(ll_times[i].number)])
             space.setitem(w_times, w_key,
-                          space.wrap(ll_times[i].counter))
+                          space.newint(ll_times[i].counter))
     w_counters = space.newdict()
     for i, counter_name in enumerate(Counters.counter_names):
         v = jit_hooks.stats_get_counter_value(None, i)
-        space.setitem_str(w_counters, counter_name, space.wrap(v))
+        space.setitem_str(w_counters, counter_name, space.newint(v))
     w_counter_times = space.newdict()
     tr_time = jit_hooks.stats_get_times_value(None, Counters.TRACING)
-    space.setitem_str(w_counter_times, 'TRACING', space.wrap(tr_time))
+    space.setitem_str(w_counter_times, 'TRACING', space.newfloat(tr_time))
     b_time = jit_hooks.stats_get_times_value(None, Counters.BACKEND)
-    space.setitem_str(w_counter_times, 'BACKEND', space.wrap(b_time))
+    space.setitem_str(w_counter_times, 'BACKEND', space.newfloat(b_time))
     return W_JitInfoSnapshot(space, w_times, w_counters, w_counter_times)
 
 def get_stats_asmmemmgr(space):
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -61,10 +61,8 @@
             if ZIPSEP != os.path.sep:
                 key = key.replace(ZIPSEP, os.path.sep)
             space.setitem(w_d, w_fs(key), space.newtuple([
-                w_fs(info.filename), space.newint(info.compress_type),
-                space.newint(info.compress_size),
-                space.newint(info.file_size), space.newint(info.file_offset),
-                space.newint(info.dostime),
+                w_fs(info.filename), space.newint(info.compress_type), space.newint(info.compress_size),
+                space.newint(info.file_size), space.newint(info.file_offset), space.newint(info.dostime),
                 space.newint(info.dosdate), space.newint(info.CRC)]))
         return w_d
 

From pypy.commits at gmail.com  Tue Feb 14 12:09:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:09:16 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: This class only ever contains an
 empty list
Message-ID: <58a339bc.1282df0a.9ac0d.8653@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90120:1e5e474fe89b
Date: 2017-02-14 17:54 +0100
http://bitbucket.org/pypy/pypy/changeset/1e5e474fe89b/

Log:	This class only ever contains an empty list

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -48,10 +48,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
+        assert fields == []
 
     def spacebind(self, space):
-        return space.newtuple([space.newtext(field) for field in self.fields])
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -443,10 +443,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
+        assert fields == []
 
     def spacebind(self, space):
-        return space.newtuple([space.newtext(field) for field in self.fields])
+        return space.newtuple([])
 
 
 class W_AST(W_Root):

From pypy.commits at gmail.com  Tue Feb 14 12:09:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:09:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 97de008631bc
Message-ID: <58a339be.0a0a1c0a.a4988.5a37@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90121:353187813fc8
Date: 2017-02-14 18:08 +0100
http://bitbucket.org/pypy/pypy/changeset/353187813fc8/

Log:	hg merge 97de008631bc

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -50,7 +50,7 @@
     def __init__(self, fields):
         assert fields == []
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
         return space.newtuple([])
 
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -479,8 +479,8 @@
     def __init__(self, fields):
         assert fields == []
 
-    def __spacebind__(self, space):
-        return space.newtuple([space.newtext(field) for field in self.fields])
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -306,7 +306,10 @@
         raise oefmt(space.w_TypeError,
                     "ord() expected string of length 1, but %T found", self)
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
+        """ Return a version of the object bound to a specific object space
+        instance. This is used for objects (like e.g. TypeDefs) that are
+        constructed before there is an object space instance. """
         return self
 
     def unwrap(self, space):
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -226,8 +226,8 @@
                     w_value.w_traceback = tb
                 else:
                     # traceback has escaped
-                    space.setattr(w_value, space.wrap("__traceback__"),
-                                  space.wrap(self.get_traceback()))
+                    space.setattr(w_value, space.newtext("__traceback__"),
+                                  self.get_w_traceback(space))
         else:
             # the only case left here is (inst, None), from a 'raise inst'.
             w_inst = w_type
@@ -270,7 +270,7 @@
                 pass
             w_t = self.w_type
             w_v = self.get_w_value(space)
-            w_tb = self.get_traceback()
+            w_tb = self.get_w_traceback(space)
             if where or objrepr:
                 if with_traceback:
                     first_line = 'From %s%s:\n' % (where, objrepr)
@@ -334,7 +334,14 @@
         else:
             self._exception_getclass(space, w_cause, "exception causes")
         w_value = self.get_w_value(space)
-        space.setattr(w_value, space.wrap("__cause__"), w_cause)
+        space.setattr(w_value, space.newtext("__cause__"), w_cause)
+
+    def get_w_traceback(self, space):
+        """Return a traceback or w_None. """
+        tb = self.get_traceback()
+        if tb is None:
+            return space.w_None
+        return tb
 
     def set_traceback(self, traceback):
         """Set the current traceback."""
@@ -369,7 +376,7 @@
         w_context = context.get_w_value(space)
         if not space.is_w(w_value, w_context):
             _break_context_cycle(space, w_value, w_context)
-            space.setattr(w_value, space.wrap('__context__'), w_context)
+            space.setattr(w_value, space.newtext('__context__'), w_context)
 
     # A simplified version of _PyErr_TrySetFromCause, which returns a
     # new exception of the same class, but with another error message.
@@ -414,11 +421,11 @@
     This is O(chain length) but context chains are usually very short
     """
     while True:
-        w_next = space.getattr(w_context, space.wrap('__context__'))
+        w_next = space.getattr(w_context, space.newtext('__context__'))
         if space.is_w(w_next, space.w_None):
             break
         if space.is_w(w_next, w_value):
-            space.setattr(w_context, space.wrap('__context__'), space.w_None)
+            space.setattr(w_context, space.newtext('__context__'), space.w_None)
             break
         w_context = w_next
 
@@ -750,6 +757,6 @@
             # when untranslated, we don't wrap into an app-level
             # SystemError (this makes debugging tests harder)
             raise
-        return OperationError(space.w_SystemError, space.wrap(
+        return OperationError(space.w_SystemError, space.newtext(
             "unexpected internal exception (please report a bug): %r%s" %
             (e, extra)))
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -312,7 +312,7 @@
                 operr.normalize_exception(space)
                 w_value = operr.get_w_value(space)
                 w_arg = space.newtuple([operr.w_type, w_value,
-                                        operr.get_traceback()])
+                                        operr.get_w_traceback(space)])
 
             d = frame.getorcreatedebug()
             if d.w_locals is not None:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1058,7 +1058,7 @@
 
     # lazy binding to space
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
         # we first make a real Function object out of it
         # and the result is a wrapped version of this Function.
         return self.get_function(space)
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -106,7 +106,7 @@
         operationerr.normalize_exception(space)
         w_type = operationerr.w_type
         w_value = operationerr.get_w_value(space)
-        w_traceback = space.wrap(operationerr.get_traceback())
+        w_traceback = operationerr.get_w_traceback(space)
 
         # for debugging convenience we also insert the exception into
         # the interpreter-level sys.last_xxx
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1226,7 +1226,7 @@
         if isinstance(w_unroller, SApplicationException):
             # app-level exception
             operr = w_unroller.operr
-            w_traceback = operr.get_traceback()
+            w_traceback = operr.get_w_traceback(self.space)
             w_res = self.call_contextmanager_exit_function(
                 w_exitfunc,
                 operr.w_type,
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -466,7 +466,7 @@
     def __init__(self, function):
         self.function = function
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
         return self.function(space)
 
 # ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/call_python.py b/pypy/module/_cffi_backend/call_python.py
--- a/pypy/module/_cffi_backend/call_python.py
+++ b/pypy/module/_cffi_backend/call_python.py
@@ -130,4 +130,4 @@
 
 @specialize.memo()
 def get_generic_decorator(space):
-    return space.wrap(interp2app(externpy_deco)) # init time
+    return interp2app(externpy_deco).spacebind(space)
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -152,7 +152,7 @@
         raise oefmt(space.w_TypeError,
                     "don't know the size pointed to by '%s'", ctype.name)
     ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
-    return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
+    return MiniBuffer(LLBuffer(ptr, size), w_cdata)
 
 MiniBuffer.typedef = TypeDef(
     "_cffi_backend.buffer",
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -183,7 +183,7 @@
                 e.normalize_exception(space)
                 w_t = e.w_type
                 w_v = e.get_w_value(space)
-                w_tb = space.wrap(e.get_traceback())
+                w_tb = e.get_w_traceback(space)
                 w_res = space.call_function(self.w_onerror, w_t, w_v, w_tb)
                 if not space.is_none(w_res):
                     self.convert_result(ll_res, w_res)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -97,7 +97,7 @@
 
     def _fget(self, attrchar):
         if attrchar == 'i':     # item
-            return self.space.wrap(self.ctitem)
+            return self.ctitem
         if attrchar == 'l':     # length
             if self.length >= 0:
                 return self.space.newint(self.length)
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -23,15 +23,15 @@
             space = self.space
             w_dct = space.newdict()
             for enumvalue, enumerator in self.enumvalues2erators.iteritems():
-                space.setitem(w_dct, space.wrap(enumvalue),
-                                     space.wrap(enumerator))
+                space.setitem(w_dct, space.newint(enumvalue),
+                                     space.newtext(enumerator))
             return w_dct
         if attrchar == 'R':     # relements
             space = self.space
             w_dct = space.newdict()
             for enumerator, enumvalue in self.enumerators2values.iteritems():
-                space.setitem(w_dct, space.wrap(enumerator),
-                                     space.wrap(enumvalue))
+                space.setitem(w_dct, space.newtext(enumerator),
+                                     space.newint(enumvalue))
             return w_dct
         return self._super._fget(self, attrchar)
 
@@ -51,7 +51,7 @@
             s = self.enumvalues2erators[value]
         except KeyError:
             s = str(value)
-        return self.space.wrap(s)
+        return self.space.newtext(s)
 
 
 class W_CTypeEnumSigned(_Mixin_Enum, W_CTypePrimitiveSigned):
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -117,12 +117,11 @@
 
     def _fget(self, attrchar):
         if attrchar == 'a':    # args
-            return self.space.newtuple([self.space.wrap(a)
-                                        for a in self.fargs])
+            return self.space.newtuple([a for a in self.fargs])
         if attrchar == 'r':    # result
             return self.ctitem
         if attrchar == 'E':    # ellipsis
-            return self.space.wrap(self.ellipsis)
+            return self.space.newbool(self.ellipsis)
         if attrchar == 'A':    # abi
             return self.space.newint(self.abi)
         return W_CTypePtrBase._fget(self, attrchar)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -262,7 +262,7 @@
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_ALL)
         align = w_ctype.alignof()
-        return self.space.wrap(align)
+        return self.space.newint(align)
 
 
     @unwrap_spec(w_name=WrappedDefault(None),
@@ -281,7 +281,7 @@
         #
         # returns a single-argument function
         space = self.space
-        w_ffi = space.wrap(self)
+        w_ffi = self
         w_decorator = call_python.get_generic_decorator(space)
         return space.appexec([w_decorator, w_ffi, w_name, w_error, w_onerror],
         """(decorator, ffi, name, error, onerror):
@@ -389,7 +389,7 @@
                 result += ')'
             result += w_ctype.name[w_ctype.name_position:]
         # Python 3: bytes -> unicode string
-        return self.space.wrap(result)
+        return self.space.newtext(result)
 
 
     @unwrap_spec(code=int)
@@ -502,7 +502,7 @@
             _, offset = w_ctype.direct_typeoffsetof(w_field_or_array, False)
         else:
             offset = self._more_offsetof(w_ctype, w_field_or_array, args_w)
-        return self.space.wrap(offset)
+        return self.space.newint(offset)
 
 
     @unwrap_spec(w_cdata=W_CData, maxlen=int)
@@ -558,7 +558,7 @@
             if size < 0:
                 raise oefmt(self.w_FFIError,
                             "don't know the size of ctype '%s'", w_ctype.name)
-        return self.space.wrap(size)
+        return self.space.newint(size)
 
 
     def descr_typeof(self, w_arg):
@@ -626,7 +626,7 @@
         lst1_w = []
         for i in range(rffi.getintfield(ctx, 'c_num_typenames')):
             s = rffi.charp2str(ctx.c_typenames[i].c_name)
-            lst1_w.append(space.wrap(s))
+            lst1_w.append(space.newtext(s))
 
         lst2_w = []
         lst3_w = []
@@ -639,7 +639,7 @@
                 lst_w = lst3_w
             else:
                 lst_w = lst2_w
-            lst_w.append(space.wrap(s))
+            lst_w.append(space.newtext(s))
 
         return space.newtuple([space.newlist(lst1_w),
                                space.newlist(lst2_w),
@@ -718,7 +718,7 @@
     return r
 
 def W_FFIObject___new__(space, w_subtype, __args__):
-    return space.wrap(make_plain_ffi_object(space, w_subtype))
+    return make_plain_ffi_object(space, w_subtype)
 
 def make_CData(space):
     return space.gettypefor(W_CData)
@@ -728,7 +728,7 @@
 
 def make_NULL(space):
     ctvoidp = newtype._new_voidp_type(space)
-    w_NULL = ctvoidp.cast(space.wrap(0))
+    w_NULL = ctvoidp.cast(space.newint(0))
     return w_NULL
 
 def make_error(space):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -56,7 +56,7 @@
 @unwrap_spec(w_ctype=ctypeobj.W_CType, following=int)
 def typeoffsetof(space, w_ctype, w_field_or_index, following=0):
     ctype, offset = w_ctype.direct_typeoffsetof(w_field_or_index, following)
-    return space.newtuple([space.wrap(ctype), space.newint(offset)])
+    return space.newtuple([ctype, space.newint(offset)])
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_cdata=cdataobj.W_CData, offset=int)
 def rawaddressof(space, w_ctype, w_cdata, offset):
@@ -99,7 +99,7 @@
             break
         key = rffi.charp2str(p)
         value = rffi.charp2str(rffi.ptradd(p, len(key) + 1))
-        space.setitem_str(w_dict, key, space.wrap(value))
+        space.setitem_str(w_dict, key, space.newtext(value))
         index += 1
 
 # ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -243,7 +243,7 @@
                 if (op == cffi_opcode.OP_GLOBAL_VAR or
                     op == cffi_opcode.OP_GLOBAL_VAR_F):
                     continue
-            names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
+            names_w.append(space.newtext(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
     def full_dict_copy(self):
@@ -252,7 +252,7 @@
         g = self.ctx.c_globals
         w_result = space.newdict()
         for i in range(total):
-            w_attr = space.wrap(rffi.charp2str(g[i].c_name))
+            w_attr = space.newtext(rffi.charp2str(g[i].c_name))
             w_value = self._get_attr(w_attr)
             space.setitem(w_result, w_attr, w_value)
         return w_result
@@ -261,7 +261,7 @@
         # rebuild a string object from 'varname', to do typechecks and
         # to force a unicode back to a plain string
         space = self.space
-        w_value = self._get_attr(space.wrap(varname))
+        w_value = self._get_attr(space.newtext(varname))
         if isinstance(w_value, cglob.W_GlobSupport):
             # regular case: a global variable
             return w_value.address()
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1254,6 +1254,20 @@
     orig_getline = linecache.getline
     try:
         linecache.getline = lambda *args: 'LINE'    # hack: speed up PyPy tests
+        sys.stderr = cStringIO.StringIO()
+        assert f(100) == 300
+        assert sys.stderr.getvalue() == ''
+        assert f(10000) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Traceback (most recent call last):
+  File "$", line $, in Zcb1
+    $
+  File "$", line $, in check_value
+    $
+ValueError: 42
+""")
+        sys.stderr = cStringIO.StringIO()
         bigvalue = 20000
         assert f(bigvalue) == -42
         assert matches(sys.stderr.getvalue(), """\
@@ -1261,6 +1275,59 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: $integer$
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    $
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -484,7 +484,8 @@
                   "backslashreplace", "surrogateescape", "surrogatepass",
                   "namereplace"):
         name = error + "_errors"
-        state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
+        state.codec_error_registry[error] = interp2app(
+                globals()[name]).spacebind(space)
 
 
 def _wrap_codec_error(space, operr, action, encoding):
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -60,7 +60,7 @@
                         break
             unwrap_value(space, write_ptr, ptr, 0, letter, w_res)
     except OperationError as e:
-        tbprint(space, e.get_traceback(),
+        tbprint(space, e.get_w_traceback(space),
                 space.newtext(e.errorstr(space)))
         # force the result to be zero
         if callback_ptr.result is not None:
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -54,7 +54,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, operror.get_traceback())
+        ptraceback[0] = make_ref(space, operror.get_w_traceback(space))
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
@@ -284,7 +284,7 @@
     operror.normalize_exception(space)
     w_type = operror.w_type
     w_value = operror.get_w_value(space)
-    w_tb = operror.get_traceback()
+    w_tb = operror.get_w_traceback(space)
 
     if rffi.cast(lltype.Signed, set_sys_last_vars):
         space.sys.setdictvalue(space, "last_type", w_type)
@@ -371,7 +371,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, operror.get_traceback())
+        ptraceback[0] = make_ref(space, operror.get_w_traceback(space))
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -256,7 +256,10 @@
             w_pathname = get_sourcefile(space, pathname)
         else:
             w_pathname = space.wrap_fsdecoded(code_w.co_filename)
-        w_cpathname = space.wrap_fsdecoded(cpathname)
+        if cpathname is not None:
+            w_cpathname = space.wrap_fsdecoded(cpathname)
+        else:
+            w_cpathname = space.w_None
         space.setitem(w_dict, space.newtext("__file__"), w_pathname)
         space.setitem(w_dict, space.newtext("__cached__"), w_cpathname)
     code_w.exec_code(space, w_dict, w_dict)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -225,7 +225,7 @@
             if operror is None:
                 return space.w_None
             else:
-                return operror.get_traceback()
+                return operror.get_w_traceback(space)
         return None
 
     def get_flag(self, name):
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -124,7 +124,7 @@
         return space.newtuple([space.w_None, space.w_None, space.w_None])
     else:
         return space.newtuple([operror.w_type, operror.get_w_value(space),
-                               operror.get_traceback()])
+                               operror.get_w_traceback(space)])
 
 def exc_info_without_tb(space, operror):
     return space.newtuple([operror.w_type, operror.get_w_value(space),
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -162,7 +162,7 @@
         if isinstance(x, float):
             return W_FloatObject(x)
         if isinstance(x, W_Root):
-            w_result = x.__spacebind__(self)
+            w_result = x.spacebind(self)
             #print 'wrapping', x, '->', w_result
             return w_result
         if isinstance(x, base_int):

From pypy.commits at gmail.com  Tue Feb 14 12:10:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:10:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 0292533aef1d
Message-ID: <58a33a1d.913f1c0a.894fb.488e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90122:9482cbbe738a
Date: 2017-02-14 18:10 +0100
http://bitbucket.org/pypy/pypy/changeset/9482cbbe738a/

Log:	hg merge 0292533aef1d

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -26,8 +26,8 @@
     elif isinstance(addr, rsocket.INET6Address):
         return space.newtuple([space.newtext(addr.get_host()),
                                space.newint(addr.get_port()),
-                               space.newint(addr.get_flowinfo()), # YYY
-                               space.newint(addr.get_scope_id())]) # YYY
+                               space.newint(addr.get_flowinfo()),
+                               space.newint(addr.get_scope_id())])
     elif rsocket.HAS_AF_PACKET and isinstance(addr, rsocket.PacketAddress):
         return space.newtuple([space.newtext(addr.get_ifname(fd)),
                                space.newint(addr.get_protocol()),
@@ -638,7 +638,7 @@
                 if value_ptr:
                     lltype.free(value_ptr, flavor='raw')
 
-            return space.newint(recv_ptr[0]) # YYY
+            return space.newint(recv_ptr[0])
         finally:
             lltype.free(recv_ptr, flavor='raw')
 

From pypy.commits at gmail.com  Tue Feb 14 12:14:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:14:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge df80a32ac30f
Message-ID: <58a33adf.8e121c0a.81a98.ad2a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90123:986a8facd2f0
Date: 2017-02-14 18:13 +0100
http://bitbucket.org/pypy/pypy/changeset/986a8facd2f0/

Log:	hg merge df80a32ac30f

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -333,7 +333,7 @@
         # obscure hack to stuff the translation options into the translated PyPy
         import pypy.module.sys
         options = make_dict(config)
-        wrapstr = 'space.newtext(%r)' % (options)
+        wrapstr = 'space.wrap(%r)' % (options) # import time
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -19,10 +19,10 @@
 
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
-    w_msg = space.wrap(msg)
+    w_msg = space.newtext(msg)
     w_filename = space.wrap_fsdecoded(fn)
-    w_lineno = space.wrap(lineno)
-    w_offset = space.wrap(offset)
+    w_lineno = space.newint(lineno)
+    w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
 
 
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -65,7 +65,7 @@
         # decode input (with the eventual \r from a previous pass)
         if not space.is_w(self.w_decoder, space.w_None):
             w_output = space.call_method(self.w_decoder, "decode",
-                                         w_input, space.newbool(final))
+                                         w_input, space.newbool(bool(final)))
         else:
             w_output = w_input
 
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -48,7 +48,7 @@
 class UMathModule(MixedModule):
     appleveldefs = {}
     interpleveldefs = {
-        'FLOATING_POINT_SUPPORT': 'space.wrap(1)',
+        'FLOATING_POINT_SUPPORT': 'space.newint(1)',
         'frompyfunc': 'ufuncs.frompyfunc',
         }
     # ufuncs
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -68,7 +68,7 @@
         scalar = multiarray.get("scalar")
 
         ret = space.newtuple([scalar, space.newtuple(
-            [space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
+            [self._get_dtype(space), space.newtext(self.raw_str())])])
         return ret
 
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -196,6 +196,9 @@
         return StringObject(obj)
     newbytes = newtext
 
+    def newunicode(self, obj):
+        raise NotImplementedError
+
     def newlist(self, items):
         return ListObject(items)
 
@@ -667,7 +670,7 @@
 
     def execute(self, interp):
         w_list = interp.space.newlist(
-            [interp.space.wrap(float(i)) for i in range(self.v)]
+            [interp.space.newfloat(float(i)) for i in range(self.v)]
         )
         dtype = get_dtype_cache(interp.space).w_float64dtype
         return array(interp.space, w_list, w_dtype=dtype, w_order=None)
@@ -834,7 +837,7 @@
                 w_res = arr.descr_take(interp.space, arg)
             elif self.name == "searchsorted":
                 w_res = arr.descr_searchsorted(interp.space, arg,
-                                               interp.space.wrap('left'))
+                                               interp.space.newtext('left'))
             else:
                 assert False # unreachable code
         elif self.name in THREE_ARG_FUNCTIONS:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -303,7 +303,7 @@
                 copy = True
             w_ret = new_view(space, orig_arr, chunks)
             if copy:
-                w_ret = w_ret.descr_copy(space, space.wrap(w_ret.get_order()))
+                w_ret = w_ret.descr_copy(space, space.newint(w_ret.get_order()))
             return w_ret
 
     def descr_setitem(self, space, orig_arr, w_index, w_value):
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -281,7 +281,7 @@
                     substr = ["(", space.str_w(space.str(
                         subdtype.subdtype.descr_get_descr(space, style='descr_subdtype'))),
                         ', ',
-                        space.str_w(space.repr(space.newtuple([space.wrap(s) for s in subdtype.shape]))),
+                        space.str_w(space.repr(space.newtuple([space.newint(s) for s in subdtype.shape]))),
                         "),"]
                 else:
                     substr = ["'", subdtype.get_str(ignore=''), "',"]
@@ -366,12 +366,12 @@
         return space.newbool(self.is_native())
 
     def descr_get_base(self, space):
-        return space.wrap(self.base)
+        return self.base
 
     def descr_get_subdtype(self, space):
         if self.subdtype is None:
             return space.w_None
-        return space.newtuple([space.wrap(self.subdtype),
+        return space.newtuple([self.subdtype,
                                self.descr_get_shape(space)])
 
     def descr_get_shape(self, space):
@@ -595,7 +595,7 @@
 
     def runpack_str(self, space, s):
         if self.is_str_or_unicode():
-            return self.coerce(space, space.wrap(s))
+            return self.coerce(space, space.newbytes(s))
         return self.itemtype.runpack_str(space, s, self.is_native())
 
     def store(self, arr, i, offset, value):
@@ -944,8 +944,8 @@
             raise
         # handle only simple cases for testing
         if space.isinstance_w(w_spec, space.w_str):
-            spec = [s.strip() for s in space.str_w(w_spec).split(',')]
-            w_lst = space.newlist([space.wrap(s) for s in spec]) 
+            spec = [s.strip() for s in space.text_w(w_spec).split(',')]
+            w_lst = space.newlist([space.newtext(s) for s in spec]) 
     if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1:
         raise oefmt(space.w_RuntimeError,
                     "_commastring is not returning a list with len >= 1")
@@ -1067,7 +1067,7 @@
     if space.isinstance_w(w_dtype, w_subtype):
         return w_dtype
     if space.isinstance_w(w_dtype, space.w_unicode):
-        name = space.str_w(w_dtype)
+        name = space.text_w(w_dtype)
         if _check_for_commastring(name):
             return _set_metadata_and_copy(space, w_metadata,
                                 dtype_from_spec(space, w_dtype, alignment))
@@ -1453,17 +1453,17 @@
         }
         w_typeinfo = space.newdict()
         for k, v in typeinfo_partial.iteritems():
-            space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v))
+            space.setitem(w_typeinfo, space.newtext(k), space.gettypefor(v))
         for k, dtype in typeinfo_full.iteritems():
             itembits = dtype.elsize * 8
             if k in ('INTP', 'UINTP'):
                 char = getattr(NPY, k + 'LTR')
             else:
                 char = dtype.char
-            items_w = [space.wrap(char),
-                       space.wrap(dtype.num),
-                       space.wrap(itembits),
-                       space.wrap(dtype.itemtype.get_element_size())]
+            items_w = [space.newtext(char),
+                       space.newint(dtype.num),
+                       space.newint(itembits),
+                       space.newint(dtype.itemtype.get_element_size())]
             if dtype.is_int():
                 if dtype.is_bool():
                     w_maxobj = space.newint(1)
@@ -1477,7 +1477,7 @@
                     w_minobj = space.newint(0)
                 items_w = items_w + [w_maxobj, w_minobj]
             items_w = items_w + [dtype.w_box_type]
-            space.setitem(w_typeinfo, space.wrap(k), space.newtuple(items_w))
+            space.setitem(w_typeinfo, space.newtext(k), space.newtuple(items_w))
         self.w_typeinfo = w_typeinfo
 
 
diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -40,14 +40,14 @@
         self.implementation = FakeArrayImplementation(self.base)
 
     def descr_base(self, space):
-        return space.wrap(self.base)
+        return self.base
 
     def descr_index(self, space):
-        return space.wrap(self.state.index)
+        return space.newint(self.state.index)
 
     def descr_coords(self, space):
         coords = self.iter.indices(self.state)
-        return space.newtuple([space.wrap(c) for c in coords])
+        return space.newtuple([space.newint(c) for c in coords])
 
     def descr_iter(self):
         return self
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -462,7 +462,7 @@
     signed = True
 
     def to_builtin_type(self, space, box):
-        return space.newint(self.for_computation(self.unbox(box)))
+        return space.wrap(self.for_computation(self.unbox(box)))
 
     def _base_coerce(self, space, w_item):
         if w_item is None:
@@ -740,7 +740,7 @@
     strlen = 32
 
     def to_builtin_type(self, space, box):
-        return space.wrap(self.for_computation(self.unbox(box)))
+        return space.newfloat(self.for_computation(self.unbox(box)))
 
     def _coerce(self, space, w_item):
         if w_item is None:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -876,13 +876,13 @@
         w_op_dtypes = space.w_None
         for tf in need_to_cast:
             if tf:
-                w_casting = space.wrap('safe')
-                w_op_dtypes = space.newtuple([space.wrap(d) for d in dtypes])
-                
+                w_casting = space.newtext('safe')
+                w_op_dtypes = space.newtuple([d for d in dtypes])
+
         w_flags = space.w_None # NOT 'external_loop', we do coalescing by core_num_dims
-        w_ro = space.newtuple([space.wrap('readonly'), space.wrap('copy')])
-        w_rw = space.newtuple([space.wrap('readwrite'), space.wrap('updateifcopy')])
-        
+        w_ro = space.newtuple([space.newtext('readonly'), space.newtext('copy')])
+        w_rw = space.newtuple([space.newtext('readwrite'), space.newtext('updateifcopy')])
+
         w_op_flags = space.newtuple([w_ro] * len(inargs) + [w_rw] * len(outargs))
         w_op_axes = space.w_None
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -90,6 +90,8 @@
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
             self._interplevel_classes[w_type] = cls
+        self.w_bytes = self.w_str
+        self.w_text = self.w_str # this is w_unicode on Py3
         self.w_dict.flag_map_or_seq = 'M'
         self.w_list.flag_map_or_seq = 'S'
         self.w_tuple.flag_map_or_seq = 'S'

From pypy.commits at gmail.com  Tue Feb 14 12:18:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:18:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge ffa1b618069f
Message-ID: <58a33bd0.da6d1c0a.2860d.b3dd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90124:ce468b82d2f3
Date: 2017-02-14 18:17 +0100
http://bitbucket.org/pypy/pypy/changeset/ce468b82d2f3/

Log:	hg merge ffa1b618069f

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -188,7 +188,7 @@
         return state.c_voidp
 
     def convert_argument(self, space, w_obj, address, call_local):
-        w_tc = space.findattr(w_obj, space.wrap('typecode'))
+        w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.str_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
                         "expected %s pointer type, but received %s",
@@ -332,7 +332,7 @@
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
-        return space.wrap(address[0])
+        return space.newbytes(address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
@@ -403,7 +403,7 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
-        return space.wrap(rffi.charp2str(charpptr[0]))
+        return space.newbytes(rffi.charp2str(charpptr[0]))
 
     def free_argument(self, space, arg, call_local):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
@@ -667,7 +667,7 @@
         # TODO: get the actual type info from somewhere ...
         address = self._get_raw_address(space, w_obj, offset)
         longptr = rffi.cast(rffi.LONGP, address)
-        return space.wrap(longptr[0])
+        return space.newlong(longptr[0])
 
 
 _converters = {}         # builtin and custom types
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -56,7 +56,7 @@
             raise NotImplementedError
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ptrval = rffi.cast(rffi.ULONG, lresult)
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.wrap(self.typecode)))
+        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.newtext(self.typecode)))
         if ptrval == 0:
             from pypy.module.cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
@@ -130,9 +130,9 @@
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ccpresult = rffi.cast(rffi.CCHARP, lresult)
         if ccpresult == rffi.cast(rffi.CCHARP, 0):
-            return space.wrap("")
+            return space.newbytes("")
         result = rffi.charp2str(ccpresult)   # TODO: make it a choice to free
-        return space.wrap(result)
+        return space.newbytes(result)
 
 
 class ConstructorExecutor(FunctionExecutor):
@@ -141,7 +141,7 @@
         from pypy.module.cppyy import interp_cppyy
         newthis = capi.c_constructor(space, cppmethod, cpptype, num_args, args)
         assert lltype.typeOf(newthis) == capi.C_OBJECT
-        return space.wrap(rffi.cast(rffi.LONG, newthis))   # really want ptrdiff_t here
+        return space.newlong(rffi.cast(rffi.LONG, newthis))   # really want ptrdiff_t here
 
 
 class InstancePtrExecutor(FunctionExecutor):
@@ -202,7 +202,7 @@
         cstr, cstr_len = capi.c_call_s(space, cppmethod, cppthis, num_args, args)
         pystr = rffi.charpsize2str(cstr, cstr_len)
         capi.c_free(space, rffi.cast(rffi.VOIDP, cstr))
-        return space.wrap(pystr)
+        return space.newbytes(pystr)
 
     def execute_libffi(self, space, cif_descr, funcaddr, buffer):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -57,7 +57,7 @@
         return arg
 
     def _wrap_object(self, space, obj):
-        return space.wrap(bool(ord(rffi.cast(rffi.CHAR, obj))))
+        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
 
     def cffi_type(self, space):
         state = space.fromcache(State)
@@ -214,7 +214,7 @@
         return r_singlefloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(float(obj))
+        return space.newfloat(float(obj))
 
     def cffi_type(self, space):
         state = space.fromcache(State)
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -43,7 +43,7 @@
             errmsg = "failed to load cdll"
         else:
             errmsg = e.msg
-        raise OperationError(space.w_RuntimeError, space.wrap(str(errmsg)))
+        raise OperationError(space.w_RuntimeError, space.newtext(str(errmsg)))
     return W_CPPLibrary(space, cdll)
 
 class State(object):
@@ -63,7 +63,7 @@
     if state.w_nullptr is None:
         from pypy.module._rawffi.interp_rawffi import unpack_simple_shape
         from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.wrap('P')))
+        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.newtext('P')))
         # TODO: fix this hack; fromaddress() will allocate memory if address
         # is null and there seems to be no way around it (ll_buffer can not
         # be touched directly)
@@ -75,7 +75,7 @@
 
 @unwrap_spec(name=str)
 def resolve_name(space, name):
-    return space.wrap(capi.c_resolve_name(space, name))
+    return space.newtext(capi.c_resolve_name(space, name))
 
 @unwrap_spec(name=str)
 def scope_byname(space, name):
@@ -121,7 +121,7 @@
     return None
 
 def std_string_name(space):
-    return space.wrap(capi.std_string_name)
+    return space.newtext(capi.std_string_name)
 
 @unwrap_spec(w_callback=W_Root)
 def set_class_generator(space, w_callback):
@@ -134,7 +134,7 @@
     state.w_fngen_callback = w_callback
 
 def register_class(space, w_pycppclass):
-    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
     cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
     # add back-end specific method pythonizations (doing this on the wrapped
     # class allows simple aliasing of methods)
@@ -443,7 +443,7 @@
             try:
                 s = self.space.str_w(args_w[i])
             except OperationError:
-                s = self.space.str_w(self.space.getattr(args_w[i], self.space.wrap('__name__')))
+                s = self.space.str_w(self.space.getattr(args_w[i], self.space.newtext('__name__')))
             s = capi.c_resolve_name(self.space, s)
             if s != self.templ_args[i]:
                 raise oefmt(self.space.w_TypeError,
@@ -549,7 +549,7 @@
         # only get here if all overloads failed ...
         errmsg = 'none of the %d overloaded methods succeeded. Full details:' % len(self.functions)
         if hasattr(self.space, "fake"):     # FakeSpace fails errorstr (see below)
-            raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+            raise OperationError(self.space.w_TypeError, self.space.newtext(errmsg))
         w_exc_type = None
         all_same_type = True
         for i in range(len(self.functions)):
@@ -573,15 +573,15 @@
                 errmsg += '    Exception: '+str(e)
 
         if all_same_type and w_exc_type is not None:
-            raise OperationError(w_exc_type, self.space.wrap(errmsg))
+            raise OperationError(w_exc_type, self.space.newtext(errmsg))
         else:
-            raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+            raise OperationError(self.space.w_TypeError, self.space.newtext(errmsg))
 
     def signature(self):
         sig = self.functions[0].signature()
         for i in range(1, len(self.functions)):
             sig += '\n'+self.functions[i].signature()
-        return self.space.wrap(sig)
+        return self.space.newtext(sig)
 
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.signature() for f in self.functions]
@@ -774,7 +774,7 @@
         return capi.c_scoped_final_name(self.space, self.handle)
 
     def get_method_names(self):
-        return self.space.newlist([self.space.wrap(name) for name in self.methods])
+        return self.space.newlist([self.space.newtext(name) for name in self.methods])
 
     def get_overload(self, name):
         try:
@@ -786,7 +786,7 @@
         return new_method
 
     def get_datamember_names(self):
-        return self.space.newlist([self.space.wrap(name) for name in self.datamembers])
+        return self.space.newlist([self.space.newtext(name) for name in self.datamembers])
 
     def get_datamember(self, name):
         try:
@@ -879,17 +879,17 @@
         alldir = []
         for i in range(capi.c_num_scopes(self.space, self)):
             sname = capi.c_scope_name(self.space, self, i)
-            if sname: alldir.append(self.space.wrap(sname))
+            if sname: alldir.append(self.space.newtext(sname))
         allmeth = {}
         for i in range(capi.c_num_methods(self.space, self)):
             idx = capi.c_method_index_at(self.space, self, i)
             mname = capi.c_method_name(self.space, self, idx)
             if mname: allmeth.setdefault(mname, 0)
         for m in allmeth.keys():
-            alldir.append(self.space.wrap(m))
+            alldir.append(self.space.newtext(m))
         for i in range(capi.c_num_datamembers(self.space, self)):
             dname = capi.c_datamember_name(self.space, self, i)
-            if dname: alldir.append(self.space.wrap(dname))
+            if dname: alldir.append(self.space.newtext(dname))
         return self.space.newlist(alldir)
         
 
@@ -977,7 +977,7 @@
         num_bases = capi.c_num_bases(self.space, self)
         for i in range(num_bases):
             base_name = capi.c_base_name(self.space, self, i)
-            bases.append(self.space.wrap(base_name))
+            bases.append(self.space.newtext(base_name))
         return self.space.newlist(bases)
 
 W_CPPClass.typedef = TypeDef(
@@ -1073,7 +1073,7 @@
 
     # allow user to determine ownership rules on a per object level
     def fget_python_owns(self, space):
-        return space.wrap(self.python_owns)
+        return space.newbool(self.python_owns)
 
     @unwrap_spec(value=bool)
     def fset_python_owns(self, space, value):
@@ -1112,7 +1112,7 @@
     def instance__eq__(self, w_other):
         # special case: if other is None, compare pointer-style
         if self.space.is_w(w_other, self.space.w_None):
-            return self.space.wrap(not self._rawobject)
+            return self.space.newbool(not self._rawobject)
 
         # get here if no class-specific overloaded operator is available, try to
         # find a global overload in gbl, in __gnu_cxx (for iterators), or in the
@@ -1143,7 +1143,7 @@
         # the first data member in a struct and the struct have the same address)
         other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False)  # TODO: factor out
         iseq = (self._rawobject == other._rawobject) and (self.cppclass == other.cppclass)
-        return self.space.wrap(iseq)
+        return self.space.newbool(iseq)
 
     def instance__ne__(self, w_other):
         return self.space.not_(self.instance__eq__(w_other))
@@ -1171,7 +1171,7 @@
         w_as_builtin = self._get_as_builtin()
         if w_as_builtin is not None:
             return self.space.repr(w_as_builtin)
-        return self.space.wrap("<%s object at 0x%x>" %
+        return self.space.newtext("<%s object at 0x%x>" %
                                (self.cppclass.name, rffi.cast(rffi.ULONG, self.get_rawobject())))
 
     def destruct(self):
@@ -1237,12 +1237,12 @@
     except KeyError:
         final_name = capi.c_scoped_final_name(space, handle)
         # the callback will cache the class by calling register_class
-        w_pycppclass = space.call_function(state.w_clgen_callback, space.wrap(final_name))
+        w_pycppclass = space.call_function(state.w_clgen_callback, space.newtext(final_name))
     return w_pycppclass
 
 def get_interface_func(space, w_callable, npar):
     state = space.fromcache(State)
-    return space.call_function(state.w_fngen_callback, w_callable, space.wrap(npar))
+    return space.call_function(state.w_fngen_callback, w_callable, space.newint(npar))
 
 def wrap_cppobject(space, rawobject, cppclass,
                    do_cast=True, python_owns=False, is_ref=False, fresh=False):
@@ -1257,7 +1257,7 @@
                 w_pycppclass = get_pythonized_cppclass(space, actual)
                 offset = capi.c_base_offset1(space, actual, cppclass, rawobject, -1)
                 rawobject = capi.direct_ptradd(rawobject, offset)
-                w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+                w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
                 cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
             except Exception:
                 # failed to locate/build the derived class, so stick to the base (note
@@ -1294,7 +1294,7 @@
 def addressof(space, w_obj):
     """Takes a bound C++ instance or array, returns the raw address."""
     address = _addressof(space, w_obj)
-    return space.wrap(address)
+    return space.newlong(address)
 
 @unwrap_spec(owns=bool, cast=bool)
 def bind_object(space, w_obj, w_pycppclass, owns=False, cast=False):
@@ -1305,7 +1305,7 @@
     except Exception:
         # accept integer value as address
         rawobject = rffi.cast(capi.C_OBJECT, space.uint_w(w_obj))
-    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
     if not w_cppclass:
         w_cppclass = scope_byname(space, space.str_w(w_pycppclass))
         if not w_cppclass:
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -18,8 +18,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct = cls.space.wrap(test_dct)
-        cls.w_capi_identity = cls.space.wrap(capi.identify())
+        cls.w_test_dct = cls.space.newtext(test_dct)
+        cls.w_capi_identity = cls.space.newtext(capi.identify())
         cls.w_advanced = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py
--- a/pypy/module/cppyy/test/test_cint.py
+++ b/pypy/module/cppyy/test/test_cint.py
@@ -169,11 +169,11 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)
-        cls.w_M = cls.space.wrap(10)
-        cls.w_fname = cls.space.wrap("test.root")
-        cls.w_tname = cls.space.wrap("test")
-        cls.w_title = cls.space.wrap("test tree")
+        cls.w_N = cls.space.newint(5)
+        cls.w_M = cls.space.newint(10)
+        cls.w_fname = cls.space.newtext("test.root")
+        cls.w_tname = cls.space.newtext("test")
+        cls.w_title = cls.space.newtext("test tree")
         cls.w_iotypes = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (iotypes_dct,))
diff --git a/pypy/module/cppyy/test/test_crossing.py b/pypy/module/cppyy/test/test_crossing.py
--- a/pypy/module/cppyy/test/test_crossing.py
+++ b/pypy/module/cppyy/test/test_crossing.py
@@ -73,7 +73,7 @@
     def setup_class(cls):
         # cppyy specific additions (note that test_dct is loaded late
         # to allow the generated extension module be loaded first)
-        cls.w_test_dct    = cls.space.wrap(test_dct)
+        cls.w_test_dct    = cls.space.newtext(test_dct)
         cls.w_pre_imports = cls.space.appexec([], """():
             import ctypes, cppyy""")    # prevents leak-checking complaints on ctypes' statics
 
@@ -105,7 +105,7 @@
             from pypy.module.imp.importing import get_so_extension
             fullmodname = os.path.join(
                 os.path.dirname(mod), name + get_so_extension(space))
-            return space.wrap(fullmodname)
+            return space.newtext(fullmodname)
 
         self.w_create_cdll = self.space.wrap(interp2app(create_cdll))
 
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -12,7 +12,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -17,8 +17,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
-        cls.w_identity = cls.space.wrap(capi.identify())
+        cls.w_test_dct  = cls.space.newtext(test_dct)
+        cls.w_identity = cls.space.newtext(capi.identify())
         cls.w_fragile = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_operators.py b/pypy/module/cppyy/test/test_operators.py
--- a/pypy/module/cppyy/test/test_operators.py
+++ b/pypy/module/cppyy/test/test_operators.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)  # should be imported from the dictionary
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_operators = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_overloads.py b/pypy/module/cppyy/test/test_overloads.py
--- a/pypy/module/cppyy/test/test_overloads.py
+++ b/pypy/module/cppyy/test/test_overloads.py
@@ -16,7 +16,7 @@
 
     def setup_class(cls):
         env = os.environ
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_overloads = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -14,7 +14,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -372,7 +372,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlvector = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -203,7 +203,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -279,8 +279,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -335,8 +335,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -444,7 +444,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy, sys
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -481,7 +481,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy, sys
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_streams.py b/pypy/module/cppyy/test/test_streams.py
--- a/pypy/module/cppyy/test/test_streams.py
+++ b/pypy/module/cppyy/test/test_streams.py
@@ -15,7 +15,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_streams = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -63,6 +63,10 @@
 class FakeBase(W_Root):
     typename = None
 
+class FakeBool(FakeBase):
+    typename = "bool"
+    def __init__(self, val):
+        self.val = val
 class FakeInt(FakeBase):
     typename = "int"
     def __init__(self, val):
@@ -153,6 +157,30 @@
             return FakeInt(int(obj))
         assert 0
 
+    @specialize.argtype(1)
+    def newbool(self, obj):
+        return FakeBool(obj)
+
+    @specialize.argtype(1)
+    def newint(self, obj):
+        return FakeInt(obj)
+
+    @specialize.argtype(1)
+    def newlong(self, obj):
+        return FakeInt(obj)
+
+    @specialize.argtype(1)
+    def newfloat(self, obj):
+        return FakeFloat(obj)
+
+    @specialize.argtype(1)
+    def newbytes(self, obj):
+        return FakeString(obj)
+
+    @specialize.argtype(1)
+    def newtext(self, obj):
+        return FakeString(obj)
+
     def float_w(self, w_obj, allow_conversion=True):
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val

From pypy.commits at gmail.com  Tue Feb 14 12:39:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:39:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 036370c1dfd7 (probably
 buggy)
Message-ID: <58a340e5.d5941c0a.a0af.b642@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90125:451198f306e0
Date: 2017-02-14 18:36 +0100
http://bitbucket.org/pypy/pypy/changeset/451198f306e0/

Log:	hg merge 036370c1dfd7 (probably buggy)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -805,6 +805,11 @@
                 raise
             return None
 
+    def wrap_none(self, w_obj):
+        if w_obj is None:
+            return self.w_None
+        return w_obj
+
     @signature(types.any(), types.bool(), returns=types.instance(W_Root))
     def newbool(self, b):
         if b:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -273,7 +273,7 @@
         self.fdel = fdel
         self.doc = doc
         self.reqcls = cls
-        self.qualname = None
+        self.w_qualname = None
         self.objclass_getter = objclass_getter
         self.use_closure = use_closure
         self.name = name if name is not None else '<generic property>'
@@ -336,19 +336,19 @@
                                                space.newtext(self.name)]))
 
     def descr_get_qualname(self, space):
-        if self.qualname is None:
-            self.qualname = self._calculate_qualname(space)
-        return self.qualname
+        if self.w_qualname is None:
+            self.w_qualname = self._calculate_qualname(space)
+        return self.w_qualname
 
     def _calculate_qualname(self, space):
         if self.reqcls is None:
-            type_qualname = u'?'
+            type_qualname = '?'
         else:
             w_type = space.gettypeobject(self.reqcls.typedef)
-            type_qualname = space.unicode_w(
-                space.getattr(w_type, space.wrap('__qualname__')))
-        qualname = u"%s.%s" % (type_qualname, self.name.decode('utf-8'))
-        return space.wrap(qualname)
+            type_qualname = space.text_w(
+                space.getattr(w_type, space.newtext('__qualname__')))
+        qualname = "%s.%s" % (type_qualname, self.name)
+        return space.newtext(qualname)
 
     def descr_get_objclass(space, property):
         if property.w_objclass is not None:
@@ -360,23 +360,11 @@
                     "generic property has no __objclass__")
 
 
-def interp_attrproperty(name, cls, doc=None):
+def interp_attrproperty(name, cls, doc=None, wrapfn=None):
     "NOT_RPYTHON: initialization-time only"
-    # YYY needs some refactoring to get rid of the wrap
+    assert wrapfn is not None
     def fget(space, obj):
-        return space.wrap(getattr(obj, name))
-    return GetSetProperty(fget, cls=cls, doc=doc)
-
-def interp_attrproperty_bytes(name, cls, doc=None):
-    "NOT_RPYTHON: initialization-time only"
-    def fget(space, obj):
-        return space.newbytes(getattr(obj, name))
-    return GetSetProperty(fget, cls=cls, doc=doc)
-
-def interp_attrproperty_fsdecode(name, cls, doc=None):
-    "NOT_RPYTHON: initialization-time only"
-    def fget(space, obj):
-        return space.fsdecode(space.newbytes(getattr(obj, name)))
+        return getattr(space, wrapfn)(getattr(obj, name))
     return GetSetProperty(fget, cls=cls, doc=doc)
 
 def interp_attrproperty_w(name, cls, doc=None):
@@ -395,10 +383,10 @@
     __get__ = interp2app(GetSetProperty.descr_property_get),
     __set__ = interp2app(GetSetProperty.descr_property_set),
     __delete__ = interp2app(GetSetProperty.descr_property_del),
-    __name__ = interp_attrproperty('name', cls=GetSetProperty),
+    __name__ = interp_attrproperty('name', cls=GetSetProperty, wrapfn="newtext_or_none"),
     __qualname__ = GetSetProperty(GetSetProperty.descr_get_qualname),
     __objclass__ = GetSetProperty(GetSetProperty.descr_get_objclass),
-    __doc__ = interp_attrproperty('doc', cls=GetSetProperty),
+    __doc__ = interp_attrproperty('doc', cls=GetSetProperty, wrapfn="newtext_or_none"),
     )
 assert not GetSetProperty.typedef.acceptable_as_base_class  # no __new__
 
@@ -451,7 +439,7 @@
     __get__ = interp2app(Member.descr_member_get),
     __set__ = interp2app(Member.descr_member_set),
     __delete__ = interp2app(Member.descr_member_del),
-    __name__ = interp_attrproperty('name', cls=Member),
+    __name__ = interp_attrproperty('name', cls=Member, wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_cls', cls=Member),
     )
 assert not Member.typedef.acceptable_as_base_class  # no __new__
@@ -581,7 +569,7 @@
 
 
 Code.typedef = TypeDef('internal-code',
-    co_name = interp_attrproperty('co_name', cls=Code),
+    co_name = interp_attrproperty('co_name', cls=Code, wrapfn="newtext_or_none"),
     co_varnames = GetSetProperty(fget_co_varnames, cls=Code),
     co_argcount = GetSetProperty(fget_co_argcount, cls=Code),
     co_kwonlyargcount = GetSetProperty(fget_zero, cls=Code),
@@ -592,7 +580,7 @@
 
 BuiltinCode.typedef = TypeDef('builtin-code',
     __reduce__   = interp2app(BuiltinCode.descr__reduce__),
-    co_name = interp_attrproperty('co_name', cls=BuiltinCode),
+    co_name = interp_attrproperty('co_name', cls=BuiltinCode, wrapfn="newtext_or_none"),
     co_varnames = GetSetProperty(fget_co_varnames, cls=BuiltinCode),
     co_argcount = GetSetProperty(fget_co_argcount, cls=BuiltinCode),
     co_kwonlyargcount = GetSetProperty(fget_zero, cls=BuiltinCode),
@@ -609,21 +597,21 @@
     __hash__ = interp2app(PyCode.descr_code__hash__),
     __reduce__ = interp2app(PyCode.descr__reduce__),
     __repr__ = interp2app(PyCode.repr),
-    co_argcount = interp_attrproperty('co_argcount', cls=PyCode),
-    co_kwonlyargcount = interp_attrproperty('co_kwonlyargcount', cls=PyCode),
-    co_nlocals = interp_attrproperty('co_nlocals', cls=PyCode),
-    co_stacksize = interp_attrproperty('co_stacksize', cls=PyCode),
-    co_flags = interp_attrproperty('co_flags', cls=PyCode),
-    co_code = interp_attrproperty_bytes('co_code', cls=PyCode),
+    co_argcount = interp_attrproperty('co_argcount', cls=PyCode, wrapfn="newint"),
+    co_kwonlyargcount = interp_attrproperty('co_kwonlyargcount', cls=PyCode, wrapfn="newint"),
+    co_nlocals = interp_attrproperty('co_nlocals', cls=PyCode, wrapfn="newint"),
+    co_stacksize = interp_attrproperty('co_stacksize', cls=PyCode, wrapfn="newint"),
+    co_flags = interp_attrproperty('co_flags', cls=PyCode, wrapfn="newint"),
+    co_code = interp_attrproperty('co_code', cls=PyCode, wrapfn="newbytes"),
     co_consts = GetSetProperty(PyCode.fget_co_consts),
     co_names = GetSetProperty(PyCode.fget_co_names),
     co_varnames = GetSetProperty(PyCode.fget_co_varnames),
     co_freevars = GetSetProperty(PyCode.fget_co_freevars),
     co_cellvars = GetSetProperty(PyCode.fget_co_cellvars),
-    co_filename = interp_attrproperty_fsdecode('co_filename', cls=PyCode),
-    co_name = interp_attrproperty('co_name', cls=PyCode),
-    co_firstlineno = interp_attrproperty('co_firstlineno', cls=PyCode),
-    co_lnotab = interp_attrproperty_bytes('co_lnotab', cls=PyCode),
+    co_filename = interp_attrproperty('co_filename', cls=PyCode, wrapfn="wrap_fsdecoded"),
+    co_name = interp_attrproperty('co_name', cls=PyCode, wrapfn="newtext"),
+    co_firstlineno = interp_attrproperty('co_firstlineno', cls=PyCode, wrapfn="newint"),
+    co_lnotab = interp_attrproperty('co_lnotab', cls=PyCode, wrapfn="newbytes"),
     __weakref__ = make_weakref_descr(PyCode),
     )
 PyCode.typedef.acceptable_as_base_class = False
@@ -798,10 +786,10 @@
     __reduce__ = interp2app(PyTraceback.descr__reduce__),
     __setstate__ = interp2app(PyTraceback.descr__setstate__),
     __dir__ = interp2app(PyTraceback.descr__dir__),
-    tb_frame = interp_attrproperty('frame', cls=PyTraceback),
-    tb_lasti = interp_attrproperty('lasti', cls=PyTraceback),
+    tb_frame = interp_attrproperty_w('frame', cls=PyTraceback),
+    tb_lasti = interp_attrproperty('lasti', cls=PyTraceback, wrapfn="newint"),
     tb_lineno = GetSetProperty(PyTraceback.descr_tb_lineno),
-    tb_next = interp_attrproperty('next', cls=PyTraceback),
+    tb_next = interp_attrproperty_w('next', cls=PyTraceback),
     )
 assert not PyTraceback.typedef.acceptable_as_base_class  # no __new__
 
@@ -819,7 +807,7 @@
                             descrmismatch='close'),
     __iter__   = interp2app(GeneratorIterator.descr__iter__,
                             descrmismatch='__iter__'),
-    gi_running = interp_attrproperty('running', cls=GeneratorIterator),
+    gi_running = interp_attrproperty('running', cls=GeneratorIterator, wrapfn="newbool"),
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
     gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
     gi_yieldfrom=interp_attrproperty_w('w_yielded_from', cls=GeneratorIterator),
@@ -843,7 +831,7 @@
                             descrmismatch='close'),
     __await__  = interp2app(Coroutine.descr__await__,
                             descrmismatch='__await__'),
-    cr_running = interp_attrproperty('running', cls=Coroutine),
+    cr_running = interp_attrproperty('running', cls=Coroutine, wrapfn="newbool"),
     cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
     cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
     cr_await   = interp_attrproperty_w('w_yielded_from', cls=Coroutine),
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -4,7 +4,7 @@
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty, interp_attrproperty_w
 
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask
@@ -343,10 +343,10 @@
 
 W_CField.typedef = TypeDef(
     '_cffi_backend.CField',
-    type = interp_attrproperty('ctype', W_CField),
-    offset = interp_attrproperty('offset', W_CField),
-    bitshift = interp_attrproperty('bitshift', W_CField),
-    bitsize = interp_attrproperty('bitsize', W_CField),
-    flags = interp_attrproperty('flags', W_CField),
+    type = interp_attrproperty_w('ctype', W_CField),
+    offset = interp_attrproperty('offset', W_CField, wrapfn="newint"),
+    bitshift = interp_attrproperty('bitshift', W_CField, wrapfn="newint"),
+    bitsize = interp_attrproperty('bitsize', W_CField, wrapfn="newint"),
+    flags = interp_attrproperty('flags', W_CField, wrapfn="newint"),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -138,8 +138,8 @@
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),
         __call__ = interp2app(W_FunctionWrapper.descr_call),
-        __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper),
-        __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper),
+        __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper, wrapfn="newtext"),
+        __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper, wrapfn="newtext"),
         __doc__ = GetSetProperty(W_FunctionWrapper.descr_get_doc),
         __get__ = interp2app(W_FunctionWrapper.descr_get),
         )
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -163,14 +163,20 @@
         '_csv.Dialect',
         __new__ = interp2app(W_Dialect___new__),
 
-        delimiter        = interp_attrproperty('delimiter', W_Dialect),
-        doublequote      = interp_attrproperty('doublequote', W_Dialect),
+        delimiter        = interp_attrproperty('delimiter', W_Dialect,
+            wrapfn='newtext'),
+        doublequote      = interp_attrproperty('doublequote', W_Dialect,
+            wrapfn='newbool'),
         escapechar       = GetSetProperty(_get_escapechar, cls=W_Dialect),
-        lineterminator   = interp_attrproperty('lineterminator', W_Dialect),
+        lineterminator   = interp_attrproperty('lineterminator', W_Dialect,
+            wrapfn='newtext'),
         quotechar        = GetSetProperty(_get_quotechar, cls=W_Dialect),
-        quoting          = interp_attrproperty('quoting', W_Dialect),
-        skipinitialspace = interp_attrproperty('skipinitialspace', W_Dialect),
-        strict           = interp_attrproperty('strict', W_Dialect),
+        quoting          = interp_attrproperty('quoting', W_Dialect,
+            wrapfn='newint'),
+        skipinitialspace = interp_attrproperty('skipinitialspace', W_Dialect,
+            wrapfn='newbool'),
+        strict           = interp_attrproperty('strict', W_Dialect,
+            wrapfn='newbool'),
 
         __doc__ = """CSV dialect
 
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -247,7 +247,8 @@
 W_Reader.typedef = TypeDef(
         '_csv.reader',
         dialect = interp_attrproperty_w('dialect', W_Reader),
-        line_num = interp_attrproperty('line_num', W_Reader),
+        line_num = interp_attrproperty('line_num', W_Reader,
+            wrapfn="newint"),
         __iter__ = interp2app(W_Reader.iter_w),
         __next__ = interp2app(W_Reader.next_w),
         __doc__ = """CSV reader
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -531,7 +531,7 @@
     _dealloc_warn = interp2app(W_FileIO._dealloc_warn_w),
     name = interp_member_w('w_name', cls=W_FileIO),
     closefd = interp_attrproperty(
-        'closefd', cls=W_FileIO,
+        'closefd', cls=W_FileIO, wrapfn="newbool",
         doc="True if the file descriptor will be closed"),
     mode = GetSetProperty(W_FileIO.descr_get_mode,
                           doc="String giving the file mode"),
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -1080,7 +1080,8 @@
     truncate = interp2app(W_TextIOWrapper.truncate_w),
     close = interp2app(W_TextIOWrapper.close_w),
 
-    line_buffering = interp_attrproperty("line_buffering", W_TextIOWrapper),
+    line_buffering = interp_attrproperty("line_buffering", W_TextIOWrapper,
+        wrapfn="newint"),
     readable = interp2app(W_TextIOWrapper.readable_w),
     writable = interp2app(W_TextIOWrapper.writable_w),
     seekable = interp2app(W_TextIOWrapper.seekable_w),
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -64,10 +64,14 @@
 W_StatsEntry.typedef = TypeDef(
     '_lsprof.StatsEntry',
     code = GetSetProperty(W_StatsEntry.get_code),
-    callcount = interp_attrproperty('callcount', W_StatsEntry),
-    reccallcount = interp_attrproperty('reccallcount', W_StatsEntry),
-    inlinetime = interp_attrproperty('it', W_StatsEntry),
-    totaltime = interp_attrproperty('tt', W_StatsEntry),
+    callcount = interp_attrproperty('callcount', W_StatsEntry,
+        wrapfn="newint"),
+    reccallcount = interp_attrproperty('reccallcount', W_StatsEntry,
+        wrapfn="newint"),
+    inlinetime = interp_attrproperty('it', W_StatsEntry,
+        wrapfn="newfloat"),
+    totaltime = interp_attrproperty('tt', W_StatsEntry,
+        wrapfn="newfloat"),
     calls = GetSetProperty(W_StatsEntry.get_calls),
     __repr__ = interp2app(W_StatsEntry.repr),
 )
@@ -91,10 +95,14 @@
 W_StatsSubEntry.typedef = TypeDef(
     '_lsprof.SubStatsEntry',
     code = GetSetProperty(W_StatsSubEntry.get_code),
-    callcount = interp_attrproperty('callcount', W_StatsSubEntry),
-    reccallcount = interp_attrproperty('reccallcount', W_StatsSubEntry),
-    inlinetime = interp_attrproperty('it', W_StatsSubEntry),
-    totaltime = interp_attrproperty('tt', W_StatsSubEntry),
+    callcount = interp_attrproperty('callcount', W_StatsSubEntry,
+        wrapfn="newint"),
+    reccallcount = interp_attrproperty('reccallcount', W_StatsSubEntry,
+        wrapfn="newint"),
+    inlinetime = interp_attrproperty('it', W_StatsSubEntry,
+        wrapfn="newfloat"),
+    totaltime = interp_attrproperty('tt', W_StatsSubEntry,
+        wrapfn="newfloat"),
     __repr__ = interp2app(W_StatsSubEntry.repr),
 )
 
diff --git a/pypy/module/_rawffi/alt/interp_ffitype.py b/pypy/module/_rawffi/alt/interp_ffitype.py
--- a/pypy/module/_rawffi/alt/interp_ffitype.py
+++ b/pypy/module/_rawffi/alt/interp_ffitype.py
@@ -105,7 +105,8 @@
 
 W_FFIType.typedef = TypeDef(
     'FFIType',
-    name = interp_attrproperty('name', W_FFIType),
+    name = interp_attrproperty('name', W_FFIType,
+        wrapfn="newtext_or_none"),
     __repr__ = interp2app(W_FFIType.repr),
     deref_pointer = interp2app(W_FFIType.descr_deref_pointer),
     sizeof = interp2app(W_FFIType.descr_sizeof),
diff --git a/pypy/module/_rawffi/alt/interp_struct.py b/pypy/module/_rawffi/alt/interp_struct.py
--- a/pypy/module/_rawffi/alt/interp_struct.py
+++ b/pypy/module/_rawffi/alt/interp_struct.py
@@ -5,7 +5,7 @@
 from rpython.rlib.rgc import must_be_light_finalizer
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty, interp_attrproperty_w
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._rawffi.alt.interp_ffitype import W_FFIType
@@ -29,9 +29,11 @@
 W_Field.typedef = TypeDef(
     'Field',
     __new__ = interp2app(descr_new_field),
-    name = interp_attrproperty('name', W_Field),
-    ffitype = interp_attrproperty('w_ffitype', W_Field),
-    offset = interp_attrproperty('offset', W_Field),
+    name = interp_attrproperty('name', W_Field,
+        wrapfn="newtext_or_none"),
+    ffitype = interp_attrproperty_w('w_ffitype', W_Field),
+    offset = interp_attrproperty('offset', W_Field,
+        wrapfn="newint"),
     )
 
 
@@ -145,7 +147,7 @@
 W__StructDescr.typedef = TypeDef(
     '_StructDescr',
     __new__ = interp2app(descr_new_structdescr),
-    ffitype = interp_attrproperty('w_ffitype', W__StructDescr),
+    ffitype = interp_attrproperty_w('w_ffitype', W__StructDescr),
     define_fields = interp2app(W__StructDescr.define_fields),
     allocate = interp2app(W__StructDescr.allocate),
     fromaddress = interp2app(W__StructDescr.fromaddress),
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -4,7 +4,7 @@
 """
 
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty_w
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._rawffi.interp_rawffi import segfault_exception
@@ -197,7 +197,7 @@
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
     __len__     = interp2app(W_ArrayInstance.getlength),
     buffer      = GetSetProperty(W_ArrayInstance.getbuffer),
-    shape       = interp_attrproperty('shape', W_ArrayInstance),
+    shape       = interp_attrproperty_w('shape', W_ArrayInstance),
     free        = interp2app(W_ArrayInstance.free),
     byptr       = interp2app(W_ArrayInstance.byptr),
     itemaddress = interp2app(W_ArrayInstance.descr_itemaddress),
@@ -221,7 +221,7 @@
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
     __len__     = interp2app(W_ArrayInstance.getlength),
     buffer      = GetSetProperty(W_ArrayInstance.getbuffer),
-    shape       = interp_attrproperty('shape', W_ArrayInstance),
+    shape       = interp_attrproperty_w('shape', W_ArrayInstance),
     byptr       = interp2app(W_ArrayInstance.byptr),
     itemaddress = interp2app(W_ArrayInstance.descr_itemaddress),
 )
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -252,7 +252,8 @@
     __new__     = interp2app(descr_new_cdll),
     ptr         = interp2app(W_CDLL.ptr),
     getaddressindll = interp2app(W_CDLL.getaddressindll),
-    name        = interp_attrproperty('name', W_CDLL),
+    name        = interp_attrproperty('name', W_CDLL,
+        wrapfn="newtext_or_none"),
     __doc__     = """ C Dynamically loaded library
 use CDLL(libname) to create a handle to a C library (the argument is processed
 the same way as dlopen processes it). On such a library you can call:
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -3,7 +3,7 @@
 """
 
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import interp_attrproperty
+from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._rawffi.interp_rawffi import segfault_exception, _MS_WINDOWS
@@ -271,8 +271,10 @@
     __call__    = interp2app(W_Structure.descr_call),
     __repr__    = interp2app(W_Structure.descr_repr),
     fromaddress = interp2app(W_Structure.fromaddress),
-    size        = interp_attrproperty('size', W_Structure),
-    alignment   = interp_attrproperty('alignment', W_Structure),
+    size        = interp_attrproperty('size', W_Structure,
+        wrapfn="newint"),
+    alignment   = interp_attrproperty('alignment', W_Structure,
+        wrapfn="newint"),
     fieldoffset = interp2app(W_Structure.descr_fieldoffset),
     fieldsize   = interp2app(W_Structure.descr_fieldsize),
     size_alignment = interp2app(W_Structure.descr_size_alignment),
@@ -383,7 +385,7 @@
     __setattr__ = interp2app(W_StructureInstance.setattr),
     buffer      = GetSetProperty(W_StructureInstance.getbuffer),
     free        = interp2app(W_StructureInstance.free),
-    shape       = interp_attrproperty('shape', W_StructureInstance),
+    shape       = interp_attrproperty_w('shape', W_StructureInstance),
     byptr       = interp2app(W_StructureInstance.byptr),
     fieldaddress= interp2app(W_StructureInstance.descr_fieldaddress),
 )
@@ -404,7 +406,7 @@
     __getattr__ = interp2app(W_StructureInstance.getattr),
     __setattr__ = interp2app(W_StructureInstance.setattr),
     buffer      = GetSetProperty(W_StructureInstance.getbuffer),
-    shape       = interp_attrproperty('shape', W_StructureInstance),
+    shape       = interp_attrproperty_w('shape', W_StructureInstance),
     byptr       = interp2app(W_StructureInstance.byptr),
     fieldaddress= interp2app(W_StructureInstance.descr_fieldaddress),
 )
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -494,9 +494,11 @@
     split        = interp2app(W_SRE_Pattern.split_w),
     sub          = interp2app(W_SRE_Pattern.sub_w),
     subn         = interp2app(W_SRE_Pattern.subn_w),
-    flags        = interp_attrproperty('flags', W_SRE_Pattern),
+    flags        = interp_attrproperty('flags', W_SRE_Pattern,
+        wrapfn="newint"),
     groupindex   = GetSetProperty(W_SRE_Pattern.fget_groupindex),
-    groups       = interp_attrproperty('num_groups', W_SRE_Pattern),
+    groups       = interp_attrproperty('num_groups', W_SRE_Pattern,
+        wrapfn="newint"),
     pattern      = interp_attrproperty_w('w_pattern', W_SRE_Pattern),
 )
 W_SRE_Pattern.typedef.acceptable_as_base_class = False
@@ -687,7 +689,7 @@
     span         = interp2app(W_SRE_Match.span_w),
     expand       = interp2app(W_SRE_Match.expand_w),
     #
-    re           = interp_attrproperty('srepat', W_SRE_Match),
+    re           = interp_attrproperty_w('srepat', W_SRE_Match),
     string       = GetSetProperty(W_SRE_Match.fget_string),
     pos          = GetSetProperty(W_SRE_Match.fget_pos),
     endpos       = GetSetProperty(W_SRE_Match.fget_endpos),
@@ -749,6 +751,6 @@
     __next__ = interp2app(W_SRE_Scanner.next_w),
     match    = interp2app(W_SRE_Scanner.match_w),
     search   = interp2app(W_SRE_Scanner.search_w),
-    pattern  = interp_attrproperty('srepat', W_SRE_Scanner),
+    pattern  = interp_attrproperty_w('srepat', W_SRE_Scanner),
 )
 W_SRE_Scanner.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, interp_attrproperty_bytes
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -543,7 +543,8 @@
     __doc__ = W_BZ2Decompressor.__doc__,
     __new__ = interp2app(descr_decompressor__new__),
     __getstate__ = interp2app(W_BZ2Decompressor.descr_getstate),
-    unused_data = interp_attrproperty_bytes("unused_data", W_BZ2Decompressor),
+    unused_data = interp_attrproperty("unused_data", W_BZ2Decompressor,
+        wrapfn="newbytes"),
     eof = GetSetProperty(W_BZ2Decompressor.eof_w),
     decompress = interp2app(W_BZ2Decompressor.decompress),
     needs_input = GetSetProperty(W_BZ2Decompressor.needs_input_w),
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -229,7 +229,8 @@
     __call__ = interp2app(cfunction_descr_call),
     __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc),
     __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject),
-    __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject),
+    __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject,
+        wrapfn="newtext_or_none"),
     )
 W_PyCFunctionObject.typedef.acceptable_as_base_class = False
 
@@ -237,7 +238,8 @@
     'method',
     __get__ = interp2app(cmethod_descr_get),
     __call__ = interp2app(cmethod_descr_call),
-    __name__ = interp_attrproperty('name', cls=W_PyCMethodObject),
+    __name__ = interp_attrproperty('name', cls=W_PyCMethodObject,
+        wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCMethodObject),
     __repr__ = interp2app(W_PyCMethodObject.descr_method_repr),
     )
@@ -247,7 +249,8 @@
     'classmethod',
     __get__ = interp2app(cclassmethod_descr_get),
     __call__ = interp2app(cmethod_descr_call),
-    __name__ = interp_attrproperty('name', cls=W_PyCClassMethodObject),
+    __name__ = interp_attrproperty('name', cls=W_PyCClassMethodObject,
+        wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass',
                                          cls=W_PyCClassMethodObject),
     __repr__ = interp2app(W_PyCClassMethodObject.descr_method_repr),
@@ -259,8 +262,10 @@
     'wrapper_descriptor',
     __call__ = interp2app(cwrapper_descr_call),
     __get__ = interp2app(cmethod_descr_get),
-    __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject),
-    __doc__ = interp_attrproperty('doc', cls=W_PyCWrapperObject),
+    __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject,
+        wrapfn="newtext_or_none"),
+    __doc__ = interp_attrproperty('doc', cls=W_PyCWrapperObject,
+        wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCWrapperObject),
     __repr__ = interp2app(W_PyCWrapperObject.descr_method_repr),
     # XXX missing: __getattribute__
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -6,8 +6,8 @@
 
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.typedef import (
-    GetSetProperty, TypeDef, interp_attrproperty, interp2app)
+from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
+        interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
@@ -102,9 +102,11 @@
     __get__ = interp2app(GetSetProperty.descr_property_get),
     __set__ = interp2app(GetSetProperty.descr_property_set),
     __delete__ = interp2app(GetSetProperty.descr_property_del),
-    __name__ = interp_attrproperty('name', cls=GetSetProperty),
+    __name__ = interp_attrproperty('name', cls=GetSetProperty,
+        wrapfn="newtext_or_none"),
     __objclass__ = GetSetProperty(GetSetProperty.descr_get_objclass),
-    __doc__ = interp_attrproperty('doc', cls=GetSetProperty),
+    __doc__ = interp_attrproperty('doc', cls=GetSetProperty,
+        wrapfn="newtext_or_none"),
     )
 assert not W_MemberDescr.typedef.acceptable_as_base_class  # no __new__
 
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -1127,12 +1127,12 @@
     __new__ = interp2app(descr__new__),
 
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
-    kind = interp_attrproperty("kind", cls=W_Dtype),
-    char = interp_attrproperty("char", cls=W_Dtype),
-    num = interp_attrproperty("num", cls=W_Dtype),
-    byteorder = interp_attrproperty("byteorder", cls=W_Dtype),
-    itemsize = interp_attrproperty("elsize", cls=W_Dtype),
-    alignment = interp_attrproperty("alignment", cls=W_Dtype),
+    kind = interp_attrproperty("kind", cls=W_Dtype, wrapfn="newtext"),
+    char = interp_attrproperty("char", cls=W_Dtype, wrapfn="newtext"),
+    num = interp_attrproperty("num", cls=W_Dtype, wrapfn="newint"),
+    byteorder = interp_attrproperty("byteorder", cls=W_Dtype, wrapfn="newtext"),
+    itemsize = interp_attrproperty("elsize", cls=W_Dtype, wrapfn="newint"),
+    alignment = interp_attrproperty("alignment", cls=W_Dtype, wrapfn="newint"),
 
     name = GetSetProperty(W_Dtype.descr_get_name),
     str = GetSetProperty(W_Dtype.descr_get_str),
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1199,8 +1199,10 @@
 
     identity = GetSetProperty(W_Ufunc.descr_get_identity),
     accumulate = interp2app(W_Ufunc.descr_accumulate),
-    nin = interp_attrproperty("nin", cls=W_Ufunc),
-    nout = interp_attrproperty("nout", cls=W_Ufunc),
+    nin = interp_attrproperty("nin", cls=W_Ufunc,
+        wrapfn="newint"),
+    nout = interp_attrproperty("nout", cls=W_Ufunc,
+        wrapfn="newint"),
     nargs = interp_attrproperty("nargs", cls=W_Ufunc),
     signature = interp_attrproperty("signature", cls=W_Ufunc),
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -199,7 +199,7 @@
     __new__ = interp2app(descr_new_resop),
     __repr__ = interp2app(WrappedOp.descr_repr),
     name = GetSetProperty(WrappedOp.descr_name),
-    offset = interp_attrproperty("offset", cls=WrappedOp),
+    offset = interp_attrproperty("offset", cls=WrappedOp, wrapfn="newint"),
 )
 WrappedOp.typedef.acceptable_as_base_class = False
 
@@ -209,8 +209,8 @@
     __new__ = interp2app(descr_new_guardop),
     __repr__ = interp2app(GuardOp.descr_repr),
     name = GetSetProperty(GuardOp.descr_name),
-    offset = interp_attrproperty("offset", cls=GuardOp),
-    hash = interp_attrproperty("hash", cls=GuardOp),
+    offset = interp_attrproperty("offset", cls=GuardOp, wrapfn="newint"),
+    hash = interp_attrproperty("hash", cls=GuardOp, wrapfn="newint"),
     )
 GuardOp.typedef.acceptable_as_base_class = False
 
@@ -226,9 +226,9 @@
     bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no,
                                  doc="offset in the bytecode"),
     call_depth = interp_attrproperty("call_depth", cls=DebugMergePoint,
-                                     doc="Depth of calls within this loop"),
+                                     doc="Depth of calls within this loop", wrapfn="newint"),
     call_id = interp_attrproperty("call_id", cls=DebugMergePoint,
-                     doc="Number of applevel function traced in this loop"),
+                     doc="Number of applevel function traced in this loop", wrapfn="newint"),
     jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name,
                      doc="Name of the jitdriver 'pypyjit' in the case "
                                     "of the main interpreter loop"),
@@ -311,7 +311,8 @@
     __doc__ = W_JitLoopInfo.__doc__,
     __new__ = interp2app(descr_new_jit_loop_info),
     jitdriver_name = interp_attrproperty('jd_name', cls=W_JitLoopInfo,
-                       doc="Name of the JitDriver, pypyjit for the main one"),
+                       doc="Name of the JitDriver, pypyjit for the main one",
+                       wrapfn="newtext"),
     greenkey = interp_attrproperty_w('w_green_key', cls=W_JitLoopInfo,
                doc="Representation of place where the loop was compiled. "
                     "In the case of the main interpreter loop, it's a triplet "
@@ -319,15 +320,19 @@
     operations = interp_attrproperty_w('w_ops', cls=W_JitLoopInfo, doc=
                                        "List of operations in this loop."),
     loop_no = interp_attrproperty('loop_no', cls=W_JitLoopInfo, doc=
-                                  "Loop cardinal number"),
+                                  "Loop cardinal number",
+                                  wrapfn="newint"),
     bridge_no = GetSetProperty(W_JitLoopInfo.descr_get_bridge_no,
                                doc="bridge number (if a bridge)"),
     type = interp_attrproperty('type', cls=W_JitLoopInfo,
-                               doc="Loop type"),
+                               doc="Loop type",
+                               wrapfn="newtext"),
     asmaddr = interp_attrproperty('asmaddr', cls=W_JitLoopInfo,
-                                  doc="Address of machine code"),
+                                  doc="Address of machine code",
+                                  wrapfn="newint"),
     asmlen = interp_attrproperty('asmlen', cls=W_JitLoopInfo,
-                                  doc="Length of machine code"),
+                                  doc="Length of machine code",
+                                  wrapfn="newint"),
     __repr__ = interp2app(W_JitLoopInfo.descr_repr),
 )
 W_JitLoopInfo.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -6,8 +6,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.typedef import (
-    TypeDef, interp_attrproperty, interp_attrproperty_bytes)
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.module.struct.formatiterator import (
     PackFormatIterator, UnpackFormatIterator
 )
@@ -172,8 +171,8 @@
 
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
-    format=interp_attrproperty_bytes("format", cls=W_Struct),
-    size=interp_attrproperty("size", cls=W_Struct),
+    format=interp_attrproperty("format", cls=W_Struct, wrapfn="newbytes"),
+    size=interp_attrproperty("size", cls=W_Struct, wrapfn="newint"),
 
     pack=interp2app(W_Struct.descr_pack),
     unpack=interp2app(W_Struct.descr_unpack),
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -331,7 +331,8 @@
 
 UCD.typedef = TypeDef("unicodedata.UCD",
                       __doc__ = "",
-                      unidata_version = interp_attrproperty('version', UCD),
+                      unidata_version = interp_attrproperty('version', UCD,
+                          wrapfn="newtext"),
                       **methods)
 
 ucd_3_2_0 = UCD(unicodedb_3_2_0)
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -1,7 +1,7 @@
 import sys
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, interp_attrproperty_bytes, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.rarithmetic import intmask, r_uint, r_uint32
 from rpython.rlib.objectmodel import keepalive_until_here
@@ -344,9 +344,9 @@
     __new__ = interp2app(Decompress___new__),
     decompress = interp2app(Decompress.decompress),
     flush = interp2app(Decompress.flush),
-    unused_data = interp_attrproperty_bytes('unused_data', Decompress),
-    unconsumed_tail = interp_attrproperty_bytes('unconsumed_tail', Decompress),
-    eof = interp_attrproperty('eof', Decompress),
+    unused_data = interp_attrproperty('unused_data', Decompress, wrapfn="newbytes"),
+    unconsumed_tail = interp_attrproperty('unconsumed_tail', Decompress, wrapfn="newbytes"),
+    eof = interp_attrproperty('eof', Decompress, wrapfn="newbool"),
     __doc__ = """decompressobj([wbits]) -- Return a decompressor object.
 
 Optional arg wbits is the window buffer size.

From pypy.commits at gmail.com  Tue Feb 14 12:39:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:39:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix
Message-ID: <58a340e8.8b811c0a.3d06b.b7c5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90126:5d34edcd4e19
Date: 2017-02-14 18:39 +0100
http://bitbucket.org/pypy/pypy/changeset/5d34edcd4e19/

Log:	fix

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -90,8 +90,6 @@
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
             self._interplevel_classes[w_type] = cls
-        self.w_bytes = self.w_str
-        self.w_text = self.w_str # this is w_unicode on Py3
         self.w_dict.flag_map_or_seq = 'M'
         self.w_list.flag_map_or_seq = 'S'
         self.w_tuple.flag_map_or_seq = 'S'

From pypy.commits at gmail.com  Tue Feb 14 12:51:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:51:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix
Message-ID: <58a343b5.025a1c0a.bfac1.b988@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90127:9ea292d316da
Date: 2017-02-14 18:51 +0100
http://bitbucket.org/pypy/pypy/changeset/9ea292d316da/

Log:	fix

diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -151,12 +151,12 @@
 def _get_escapechar(space, dialect):
     if dialect.escapechar == u'\0':
         return space.w_None
-    return space.newtext(dialect.escapechar)
+    return space.newunicode(dialect.escapechar)
 
 def _get_quotechar(space, dialect):
     if dialect.quotechar == u'\0':
         return space.w_None
-    return space.newtext(dialect.quotechar)
+    return space.newunicode(dialect.quotechar)
 
 
 W_Dialect.typedef = TypeDef(
@@ -164,12 +164,12 @@
         __new__ = interp2app(W_Dialect___new__),
 
         delimiter        = interp_attrproperty('delimiter', W_Dialect,
-            wrapfn='newtext'),
+            wrapfn='newunicode'),
         doublequote      = interp_attrproperty('doublequote', W_Dialect,
             wrapfn='newbool'),
         escapechar       = GetSetProperty(_get_escapechar, cls=W_Dialect),
         lineterminator   = interp_attrproperty('lineterminator', W_Dialect,
-            wrapfn='newtext'),
+            wrapfn='newunicode'),
         quotechar        = GetSetProperty(_get_quotechar, cls=W_Dialect),
         quoting          = interp_attrproperty('quoting', W_Dialect,
             wrapfn='newint'),
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -44,9 +44,9 @@
         field = field_builder.build()
         if self.numeric_field:
             self.numeric_field = False
-            w_obj = space.call_function(space.w_float, space.newtext(field))
+            w_obj = space.call_function(space.w_float, space.newunicode(field))
         else:
-            w_obj = space.newtext(field)
+            w_obj = space.newunicode(field)
         self.fields_w.append(w_obj)
 
     def next_w(self):
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -115,7 +115,7 @@
         rec.append(dialect.lineterminator)
 
         line = rec.build()
-        return space.call_function(self.w_filewrite, space.newtext(line))
+        return space.call_function(self.w_filewrite, space.newunicode(line))
 
     def writerows(self, w_seqseq):
         """Construct and write a series of sequences to a csv file.

From pypy.commits at gmail.com  Tue Feb 14 12:55:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:55:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix
Message-ID: <58a344aa.838edf0a.f2ee1.9542@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90128:c944c1708d09
Date: 2017-02-14 18:55 +0100
http://bitbucket.org/pypy/pypy/changeset/c944c1708d09/

Log:	fix

diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -46,5 +46,5 @@
     module.setdictvalue(space, 'ffi', ffi)
     module.setdictvalue(space, 'lib', lib)
     w_modules_dict = space.sys.get('modules')
-    space.setitem(w_modules_dict, w_name, space.newtext(module))
-    space.setitem(w_modules_dict, space.newtext(name + '.lib'), space.newtext(lib))
+    space.setitem(w_modules_dict, w_name, module)
+    space.setitem(w_modules_dict, space.newtext(name + '.lib'), lib)

From pypy.commits at gmail.com  Tue Feb 14 12:58:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 09:58:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge be088827c1c4
Message-ID: <58a3453b.d8301c0a.4d323.6978@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90129:3ad4f6634212
Date: 2017-02-14 18:57 +0100
http://bitbucket.org/pypy/pypy/changeset/3ad4f6634212/

Log:	hg merge be088827c1c4

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -225,6 +225,9 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
+    def _wrap_object(self, space, obj):
+        return getattr(space, self.wrapper)(obj)
+
     def convert_argument_libffi(self, space, w_obj, address, call_local):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
@@ -236,7 +239,7 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
-        return space.wrap(rffiptr[0])
+        return self._wrap_object(space, rffiptr[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = self._get_raw_address(space, w_obj, offset)
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -81,7 +81,7 @@
     _mixin_ = True
 
     def _wrap_object(self, space, obj):
-        return space.wrap(obj)
+        return getattr(space, self.wrapper)(obj)
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
@@ -105,7 +105,7 @@
         self.do_assign = True
 
     def _wrap_object(self, space, obj):
-        return space.wrap(rffi.cast(self.c_type, obj))
+        return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -41,6 +41,17 @@
         self.c_size_t    = nt.new_primitive_type(space, 'size_t')
         self.c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
 
+class BaseIntTypeMixin(object):
+    _mixin_     = True
+    _immutable_fields_ = ['wrapper']
+
+    wrapper     = 'newint'
+
+class BaseLongTypeMixin(object):
+    _mixin_     = True
+    _immutable_fields_ = ['wrapper']
+
+    wrapper     = 'newlong'
 
 class BoolTypeMixin(object):
     _mixin_     = True
@@ -69,6 +80,7 @@
 
     c_type      = rffi.CHAR
     c_ptrtype   = rffi.CCHARP           # there's no such thing as rffi.CHARP
+    wrapper     = 'newbytes'
 
     def _unwrap_object(self, space, w_value):
         # allow int to pass to char and make sure that str is of length 1
@@ -90,7 +102,7 @@
         state = space.fromcache(State)
         return state.c_char
 
-class ShortTypeMixin(object):
+class ShortTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -104,7 +116,7 @@
         state = space.fromcache(State)
         return state.c_short
 
-class UShortTypeMixin(object):
+class UShortTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -118,7 +130,7 @@
         state = space.fromcache(State)
         return state.c_ushort
 
-class IntTypeMixin(object):
+class IntTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -132,7 +144,7 @@
         state = space.fromcache(State)
         return state.c_int
 
-class UIntTypeMixin(object):
+class UIntTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -146,7 +158,7 @@
         state = space.fromcache(State)
         return state.c_uint
 
-class LongTypeMixin(object):
+class LongTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -160,7 +172,9 @@
         state = space.fromcache(State)
         return state.c_long
 
-class ULongTypeMixin(object):
+# TODO: check ULong limits; actually, they fail if this is
+#  an BaseLongTypeMixin (i.e. use of space.ewlong) ... why??
+class ULongTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -174,7 +188,7 @@
         state = space.fromcache(State)
         return state.c_ulong
 
-class LongLongTypeMixin(object):
+class LongLongTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -188,7 +202,7 @@
         state = space.fromcache(State)
         return state.c_llong
 
-class ULongLongTypeMixin(object):
+class ULongLongTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
@@ -226,6 +240,7 @@
 
     c_type      = rffi.DOUBLE
     c_ptrtype   = rffi.DOUBLEP
+    wrapper     = 'newfloat'
     typecode    = 'd'
 
     def _unwrap_object(self, space, w_obj):
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -2,7 +2,7 @@
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty, interp_attrproperty_w
 from pypy.interpreter.baseobjspace import W_Root
 
 from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
@@ -982,7 +982,7 @@
 
 W_CPPClass.typedef = TypeDef(
     'CPPClass',
-    type_name = interp_attrproperty('name', W_CPPClass),
+    type_name = interp_attrproperty('name', W_CPPClass, wrapfn="newtext"),
     get_base_names = interp2app(W_CPPClass.get_base_names),
     get_method_names = interp2app(W_CPPClass.get_method_names),
     get_overload = interp2app(W_CPPClass.get_overload, unwrap_spec=['self', str]),
@@ -1009,7 +1009,7 @@
 
 W_ComplexCPPClass.typedef = TypeDef(
     'ComplexCPPClass',
-    type_name = interp_attrproperty('name', W_CPPClass),
+    type_name = interp_attrproperty('name', W_CPPClass, wrapfn="newtext"),
     get_base_names = interp2app(W_ComplexCPPClass.get_base_names),
     get_method_names = interp2app(W_ComplexCPPClass.get_method_names),
     get_overload = interp2app(W_ComplexCPPClass.get_overload, unwrap_spec=['self', str]),
@@ -1186,7 +1186,7 @@
 
 W_CPPInstance.typedef = TypeDef(
     'CPPInstance',
-    cppclass = interp_attrproperty('cppclass', cls=W_CPPInstance),
+    cppclass = interp_attrproperty_w('cppclass', cls=W_CPPInstance),
     _python_owns = GetSetProperty(W_CPPInstance.fget_python_owns, W_CPPInstance.fset_python_owns),
     __init__ = interp2app(W_CPPInstance.instance__init__),
     __eq__ = interp2app(W_CPPInstance.instance__eq__),
diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -288,6 +288,8 @@
     def test14_double_enum_trouble(self):
         """Test a redefinition of enum in a derived class"""
 
+        return # don't bother; is fixed in cling-support
+
         import cppyy
 
         M = cppyy.gbl.fragile.M
diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -261,6 +261,8 @@
     def test03_string_with_null_character(self):
         """Test that strings with NULL do not get truncated"""
 
+        return # don't bother; is fixed in cling-support
+
         import cppyy
         std = cppyy.gbl.std
         stringy_class = cppyy.gbl.stringy_class
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -71,6 +71,10 @@
     typename = "int"
     def __init__(self, val):
         self.val = val
+class FakeLong(FakeBase):
+    typename = "long"
+    def __init__(self, val):
+        self.val = val
 class FakeFloat(FakeBase):
     typename = "float"
     def __init__(self, val):
@@ -145,16 +149,7 @@
     def issequence_w(self, w_obj):
         return True
 
-    @specialize.argtype(1)
     def wrap(self, obj):
-        if isinstance(obj, int):
-            return FakeInt(obj)
-        if isinstance(obj, float):
-            return FakeFloat(obj)
-        if isinstance(obj, str):
-            return FakeString(obj)
-        if isinstance(obj, rffi.r_int):
-            return FakeInt(int(obj))
         assert 0
 
     @specialize.argtype(1)
@@ -163,11 +158,11 @@
 
     @specialize.argtype(1)
     def newint(self, obj):
-        return FakeInt(obj)
+        return FakeInt(rarithmetic.intmask(obj))
 
     @specialize.argtype(1)
     def newlong(self, obj):
-        return FakeInt(obj)
+        return FakeLong(rarithmetic.intmask(obj))
 
     @specialize.argtype(1)
     def newfloat(self, obj):
@@ -207,7 +202,9 @@
         return w_obj.val
 
     def uint_w(self, w_obj):
-        assert isinstance(w_obj, FakeInt)
+        if isinstance(w_obj, FakeInt):
+            return rarithmetic.r_uint(w_obj.val)
+        assert isinstance(w_obj, FakeLong)
         return rarithmetic.r_uint(w_obj.val)
 
     def str_w(self, w_obj):

From pypy.commits at gmail.com  Tue Feb 14 13:00:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 10:00:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge dedd16a4f65a
Message-ID: <58a345db.c53f1c0a.37189.6c14@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90130:6d55b94fef3a
Date: 2017-02-14 18:59 +0100
http://bitbucket.org/pypy/pypy/changeset/6d55b94fef3a/

Log:	hg merge dedd16a4f65a

diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -244,8 +244,10 @@
         class W_C(W_A):
             b = 3
         W_A.typedef = typedef.TypeDef("A",
-            a = typedef.interp_attrproperty("a", cls=W_A),
-            b = typedef.interp_attrproperty("b", cls=W_A),
+            a = typedef.interp_attrproperty("a", cls=W_A,
+                wrapfn="newint"),
+            b = typedef.interp_attrproperty("b", cls=W_A,
+                wrapfn="newint"),
         )
         class W_B(W_Root):
             pass
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -109,7 +109,7 @@
         raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
 def GetLastError(space):
-    return space.wrap(rwin32.GetLastError_saved())
+    return space.newint(rwin32.GetLastError_saved())
 
 # __________________________________________________________
 # functions for the "win32" namespace
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -625,7 +625,7 @@
 if _MS_WINDOWS:
     @unwrap_spec(code=int)
     def FormatError(space, code):
-        return space.wrap(rwin32.FormatError(code))
+        return space.newtext(rwin32.FormatError(code))
 
     @unwrap_spec(hresult=int)
     def check_HRESULT(space, hresult):
@@ -648,7 +648,7 @@
     # see also
     # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
     def get_last_error(space):
-        return space.wrap(rwin32.GetLastError_alt_saved())
+        return space.newint(rwin32.GetLastError_alt_saved())
     @unwrap_spec(error=int)
     def set_last_error(space, error):
         rwin32.SetLastError_alt_saved(error)
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -10,8 +10,8 @@
 def raiseWindowsError(space, errcode, context):
     message = rwin32.FormatError(errcode)
     raise OperationError(space.w_WindowsError,
-                         space.newtuple([space.wrap(errcode),
-                                         space.wrap(message)]))
+                         space.newtuple([space.newint(errcode),
+                                         space.newtext(message)]))
 
 class W_HKEY(W_Root):
     def __init__(self, space, hkey):
@@ -26,16 +26,16 @@
         return rffi.cast(rffi.SIZE_T, self.hkey)
 
     def descr_bool(self, space):
-        return space.wrap(self.as_int() != 0)
+        return space.newbool(self.as_int() != 0)
 
     def descr_handle_get(self, space):
-        return space.wrap(self.as_int())
+        return space.newint(self.as_int())
 
     def descr_repr(self, space):
-        return space.wrap("<PyHKEY:0x%x>" % (self.as_int(),))
+        return space.newtext("<PyHKEY:0x%x>" % (self.as_int(),))
 
     def descr_int(self, space):
-        return space.wrap(self.as_int())
+        return space.newint(self.as_int())
 
     def descr__enter__(self, space):
         return self
@@ -61,12 +61,12 @@
 On 64 bit windows, the result of this function is a long integer"""
         key = self.as_int()
         self.hkey = rwin32.NULL_HANDLE
-        return space.wrap(key)
+        return space.newint(key)
 
 @unwrap_spec(key=int)
 def new_HKEY(space, w_subtype, key):
     hkey = rffi.cast(rwinreg.HKEY, key)
-    return space.wrap(W_HKEY(space, hkey))
+    return W_HKEY(space, hkey)
 descr_HKEY_new = interp2app(new_HKEY)
 
 W_HKEY.typedef = TypeDef(
@@ -260,7 +260,7 @@
                 if ret != 0:
                     raiseWindowsError(space, ret, 'RegQueryValue')
                 length = intmask(bufsize_p[0] - 1)
-                return space.wrap(rffi.charp2strn(buf, length))
+                return space.newtext(rffi.charp2strn(buf, length))
 
 def convert_to_regdata(space, w_value, typ):
     buf = None
@@ -284,7 +284,7 @@
         else:
             if space.isinstance_w(w_value, space.w_unicode):
                 w_value = space.call_method(w_value, 'encode',
-                                            space.wrap('mbcs'))
+                                            space.newtext('mbcs'))
             buf = rffi.str2charp(space.str_w(w_value))
             buflen = space.len_w(w_value) + 1
 
@@ -304,7 +304,7 @@
                     w_item = space.next(w_iter)
                     if space.isinstance_w(w_item, space.w_unicode):
                         w_item = space.call_method(w_item, 'encode',
-                                                   space.wrap('mbcs'))
+                                                   space.newtext('mbcs'))
                     item = space.str_w(w_item)
                     strings.append(item)
                     buflen += len(item) + 1
@@ -351,9 +351,9 @@
 def convert_from_regdata(space, buf, buflen, typ):
     if typ == rwinreg.REG_DWORD:
         if not buflen:
-            return space.wrap(0)
+            return space.newint(0)
         d = rffi.cast(rwin32.LPDWORD, buf)[0]
-        return space.wrap(d)
+        return space.newint(d)
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
@@ -364,8 +364,8 @@
             if buf[buflen - 1] == '\x00':
                 buflen -= 1
             s = rffi.charp2strn(buf, buflen)
-        w_s = space.wrap(s)
-        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
+        w_s = space.newbytes(s)
+        return space.call_method(w_s, 'decode', space.newtext('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -380,7 +380,7 @@
             if len(s) == 0:
                 break
             s = ''.join(s)
-            l.append(space.wrap(s))
+            l.append(space.newtext(s))
             i += 1
         return space.newlist(l)
 
@@ -465,7 +465,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(intmask(retType[0])),
+                        space.newint(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -486,7 +486,7 @@
         ret = rwinreg.RegCreateKey(hkey, subkey, rethkey)
         if ret != 0:
             raiseWindowsError(space, ret, 'CreateKey')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(sub_key=str, reserved=int, access=rffi.r_uint)
 def CreateKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WRITE):
@@ -508,7 +508,7 @@
                                      lltype.nullptr(rwin32.LPDWORD.TO))
         if ret != 0:
             raiseWindowsError(space, ret, 'CreateKeyEx')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(subkey=str)
 def DeleteKey(space, w_hkey, subkey):
@@ -555,7 +555,7 @@
         ret = rwinreg.RegOpenKeyEx(hkey, sub_key, reserved, access, rethkey)
         if ret != 0:
             raiseWindowsError(space, ret, 'RegOpenKeyEx')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(index=int)
 def EnumValue(space, w_hkey, index):
@@ -614,10 +614,10 @@
 
                             length = intmask(retDataSize[0])
                             return space.newtuple([
-                                space.wrap(rffi.charp2str(valuebuf)),
+                                space.newtext(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(intmask(retType[0])),
+                                space.newint(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
@@ -647,7 +647,7 @@
                                        lltype.nullptr(rwin32.PFILETIME.TO))
             if ret != 0:
                 raiseWindowsError(space, ret, 'RegEnumKeyEx')
-            return space.wrap(rffi.charp2str(buf))
+            return space.newtext(rffi.charp2str(buf))
 
 def QueryInfoKey(space, w_hkey):
     """tuple = QueryInfoKey(key) - Returns information about a key.
@@ -673,9 +673,9 @@
                     raiseWindowsError(space, ret, 'RegQueryInfoKey')
                 l = ((lltype.r_longlong(ft[0].c_dwHighDateTime) << 32) +
                      lltype.r_longlong(ft[0].c_dwLowDateTime))
-                return space.newtuple([space.wrap(nSubKeys[0]),
-                                       space.wrap(nValues[0]),
-                                       space.wrap(l)])
+                return space.newtuple([space.newint(nSubKeys[0]),
+                                       space.newint(nValues[0]),
+                                       space.newint(l)])
 
 def ConnectRegistry(space, w_machine, w_hkey):
     """key = ConnectRegistry(computer_name, key)
@@ -694,13 +694,13 @@
         ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)
         if ret != 0:
             raiseWindowsError(space, ret, 'RegConnectRegistry')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(source=unicode)
 def ExpandEnvironmentStrings(space, source):
     "string = ExpandEnvironmentStrings(string) - Expand environment vars."
     try:
-        return space.wrap(rwinreg.ExpandEnvironmentStrings(source))
+        return space.newtext(rwinreg.ExpandEnvironmentStrings(source))
     except WindowsError as e:
         raise wrap_windowserror(space, e)
 

From pypy.commits at gmail.com  Tue Feb 14 13:01:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 10:01:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 511d80a64542
Message-ID: <58a3460d.88bb1c0a.d578.7490@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90131:8d50fdd3bda5
Date: 2017-02-14 19:01 +0100
http://bitbucket.org/pypy/pypy/changeset/8d50fdd3bda5/

Log:	hg merge 511d80a64542

diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py
--- a/pypy/module/_md5/interp_md5.py
+++ b/pypy/module/_md5/interp_md5.py
@@ -23,7 +23,7 @@
         return self.space.newbytes(self.digest())
 
     def hexdigest_w(self):
-        return self.space.wrap(self.hexdigest())
+        return self.space.newtext(self.hexdigest())
 
     def copy_w(self):
         clone = W_MD5(self.space)
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -70,7 +70,7 @@
         nullarr = arr.fromaddress(space, rffi.cast(rffi.ULONG, 0), 0)
         assert isinstance(nullarr, W_ArrayInstance)
         nullarr.free(space)
-        state.w_nullptr = space.wrap(nullarr)
+        state.w_nullptr = nullarr
     return state.w_nullptr
 
 @unwrap_spec(name=str)
@@ -1092,7 +1092,7 @@
 
     def _get_as_builtin(self):
         try:
-            return self.space.call_method(self.space.wrap(self), "_cppyy_as_builtin")
+            return self.space.call_method(self, "_cppyy_as_builtin")
         except OperationError as e:
             if not (e.match(self.space, self.space.w_TypeError) or
                     e.match(self.space, self.space.w_AttributeError)):
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -39,5 +39,5 @@
         space = self.space
         pypyjitdriver.space = space
         w_obj = space.wrap(PARAMETERS)
-        space.setattr(space.wrap(self), space.wrap('defaults'), w_obj)
+        space.setattr(self, space.newtext('defaults'), w_obj)
         pypy_hooks.space = space

From pypy.commits at gmail.com  Wed Feb 15 01:34:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 22:34:55 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: publicity for the leysin sprint
Message-ID: <58a3f68f.862f1c0a.bf62c.f597@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5780:bef93e5951ca
Date: 2017-02-15 07:34 +0100
http://bitbucket.org/pypy/extradoc/changeset/bef93e5951ca/

Log:	publicity for the leysin sprint

diff --git a/talk/swisspython2017/revdb/chalet1_004.jpg b/talk/swisspython2017/revdb/chalet1_004.jpg
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..65a88138b7288b8c3d9af4a2ea65ef10d0dc574d
GIT binary patch

[cut]

diff --git a/talk/swisspython2017/revdb/talk.pdf b/talk/swisspython2017/revdb/talk.pdf
index 0b5bbf46bda5af01898f6b0594e0d5dd76055487..3e16ee4941ca4a1e12f91c4668ead4cae1286f89
GIT binary patch

[cut]

diff --git a/talk/swisspython2017/revdb/talk.rst b/talk/swisspython2017/revdb/talk.rst
--- a/talk/swisspython2017/revdb/talk.rst
+++ b/talk/swisspython2017/revdb/talk.rst
@@ -156,6 +156,19 @@
 
 https://bitbucket.org/pypy/revdb/
 
+.. raw:: latex
+
+    \bigskip
+    \bigskip
+    \bigskip
+
+.. image:: chalet1_004.jpg
+   :scale: 20%
+   :align: center
+
+*Feb. 26th - March 4th: general PyPy sprint in Leysin, Vaud, Switzerland
+(everybody welcome)*
+
 
 How does it work?  (slide 1/2)
 ==============================

From pypy.commits at gmail.com  Wed Feb 15 02:31:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:31:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 2e449c823765
Message-ID: <58a403e4.cfc41c0a.db7b2.06de@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90132:0d12eeeab7ce
Date: 2017-02-15 08:10 +0100
http://bitbucket.org/pypy/pypy/changeset/0d12eeeab7ce/

Log:	hg merge 2e449c823765

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -71,7 +71,7 @@
             if self.__class__ is not OperationError and s is None:
                 s = self._compute_value(space)
             try:
-                s = space.str_w(s)
+                s = space.text_w(s)
             except Exception:
                 pass
         return '[%s: %s]' % (self.w_type, s)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1203,8 +1203,10 @@
         wrapfn="newint"),
     nout = interp_attrproperty("nout", cls=W_Ufunc,
         wrapfn="newint"),
-    nargs = interp_attrproperty("nargs", cls=W_Ufunc),
-    signature = interp_attrproperty("signature", cls=W_Ufunc),
+    nargs = interp_attrproperty("nargs", cls=W_Ufunc,
+        wrapfn="newint"),
+    signature = interp_attrproperty("signature", cls=W_Ufunc,
+        wrapfn="newtext"),
 
     reduce = interp2app(W_Ufunc.descr_reduce),
     outer = interp2app(W_Ufunc.descr_outer),

From pypy.commits at gmail.com  Wed Feb 15 02:31:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:31:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 6673bc3db17b
Message-ID: <58a403e6.d18bdf0a.1f8e0.2bf7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90133:d09311567d06
Date: 2017-02-15 08:13 +0100
http://bitbucket.org/pypy/pypy/changeset/d09311567d06/

Log:	hg merge 6673bc3db17b

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -941,7 +941,7 @@
                                              space.bytes_w(w_chunk))
 
             w_decoded = space.call_method(self.w_decoder, "decode",
-                                          w_chunk, space.newbool(cookie.need_eof))
+                                          w_chunk, space.newbool(bool(cookie.need_eof)))
             check_decoded(space, w_decoded)
             self._set_decoded_chars(space.unicode_w(w_decoded))
 
diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py
--- a/pypy/module/_pypyjson/interp_encoder.py
+++ b/pypy/module/_pypyjson/interp_encoder.py
@@ -79,4 +79,4 @@
                 sb.append(HEX[s2 & 0x0f])
 
     res = sb.build()
-    return space.wrap(res)
+    return space.newtext(res)
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -342,7 +342,7 @@
         return space.newint(address_as_uint)
 
     def getidentifier(self, space):
-        return space.wrap(self.cdll.getidentifier())
+        return space.newint(self.cdll.getidentifier())
 
 @unwrap_spec(name='str_or_None', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -91,7 +91,7 @@
         retval_w = []
         for fd, revents in retval:
             retval_w.append(space.newtuple([space.newint(fd),
-                                            space.newtext(revents)]))
+                                            space.newint(revents)]))
         return space.newlist(retval_w)
 
 pollmethods = {}
diff --git a/pypy/module/token/__init__.py b/pypy/module/token/__init__.py
--- a/pypy/module/token/__init__.py
+++ b/pypy/module/token/__init__.py
@@ -7,7 +7,7 @@
 
     appleveldefs = {}
     interpleveldefs = {
-        "NT_OFFSET" : "space.wrap(256)",
+        "NT_OFFSET" : "space.newint(256)",
         "ISTERMINAL" : "__init__.isterminal",
         "ISNONTERMINAL" : "__init__.isnonterminal",
         "ISEOF" : "__init__.iseof"
@@ -34,12 +34,12 @@
 
 @unwrap_spec(tok=int)
 def isterminal(space, tok):
-    return space.wrap(tok < 256)
+    return space.newbool(tok < 256)
 
 @unwrap_spec(tok=int)
 def isnonterminal(space, tok):
-    return space.wrap(tok >= 256)
+    return space.newbool(tok >= 256)
 
 @unwrap_spec(tok=int)
 def iseof(space, tok):
-    return space.wrap(tok == pygram.tokens.ENDMARKER)
+    return space.newbool(tok == pygram.tokens.ENDMARKER)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1980,6 +1980,7 @@
     _none_value = None
 
     def wrap(self, stringval):
+        assert stringval is not None
         return self.space.newunicode(stringval)
 
     def unwrap(self, w_string):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -376,6 +376,7 @@
         return self.newtext(s)
 
     def newunicode(self, uni):
+        assert uni is not None
         assert isinstance(uni, unicode)
         return W_UnicodeObject(uni)
 
diff --git a/pypy/objspace/std/transparent.py b/pypy/objspace/std/transparent.py
--- a/pypy/objspace/std/transparent.py
+++ b/pypy/objspace/std/transparent.py
@@ -39,9 +39,9 @@
 def setup(space):
     """Add proxy functions to the __pypy__ module."""
     w___pypy__ = space.getbuiltinmodule("__pypy__")
-    space.setattr(w___pypy__, space.newtext('tproxy'), space.wrap(app_proxy))
+    space.setattr(w___pypy__, space.newtext('tproxy'), app_proxy.spacebind(space))
     space.setattr(w___pypy__, space.newtext('get_tproxy_controller'),
-                  space.wrap(app_proxy_controller))
+                  app_proxy_controller.spacebind(space))
 
 
 def proxy(space, w_type, w_controller):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1,7 +1,8 @@
 """The builtin str implementation"""
 
 from rpython.rlib.objectmodel import (
-    compute_hash, compute_unique_id, import_from_mixin)
+    compute_hash, compute_unique_id, import_from_mixin,
+    enforceargs)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
@@ -30,6 +31,7 @@
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
 
+    @enforceargs(uni=unicode)
     def __init__(self, unistr):
         assert isinstance(unistr, unicode)
         self._value = unistr

From pypy.commits at gmail.com  Wed Feb 15 02:31:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:31:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 8dfca1a083c6
Message-ID: <58a403e8.2da9df0a.fcf9b.30a5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90134:78d15c73cfad
Date: 2017-02-15 08:16 +0100
http://bitbucket.org/pypy/pypy/changeset/78d15c73cfad/

Log:	hg merge 8dfca1a083c6

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -225,9 +225,6 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def _wrap_object(self, space, obj):
-        return getattr(space, self.wrapper)(obj)
-
     def convert_argument_libffi(self, space, w_obj, address, call_local):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -80,8 +80,8 @@
 class NumericExecutorMixin(object):
     _mixin_ = True
 
-    def _wrap_object(self, space, obj):
-        return getattr(space, self.wrapper)(obj)
+    #def _wrap_object(self, space, obj):
+    #    return getattr(space, self.wrapper)(obj)
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
@@ -104,8 +104,8 @@
         self.item = self._unwrap_object(space, w_item)
         self.do_assign = True
 
-    def _wrap_object(self, space, obj):
-        return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
+    #def _wrap_object(self, space, obj):
+    #    return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -41,18 +41,6 @@
         self.c_size_t    = nt.new_primitive_type(space, 'size_t')
         self.c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
 
-class BaseIntTypeMixin(object):
-    _mixin_     = True
-    _immutable_fields_ = ['wrapper']
-
-    wrapper     = 'newint'
-
-class BaseLongTypeMixin(object):
-    _mixin_     = True
-    _immutable_fields_ = ['wrapper']
-
-    wrapper     = 'newlong'
-
 class BoolTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
@@ -60,6 +48,9 @@
     c_type      = rffi.UCHAR
     c_ptrtype   = rffi.UCHARP
 
+    def _wrap_object(self, space, obj):
+        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
+
     def _unwrap_object(self, space, w_obj):
         arg = space.c_int_w(w_obj)
         if arg != False and arg != True:
@@ -80,7 +71,9 @@
 
     c_type      = rffi.CHAR
     c_ptrtype   = rffi.CCHARP           # there's no such thing as rffi.CHARP
-    wrapper     = 'newbytes'
+
+    def _wrap_object(self, space, obj):
+        return space.newbytes(obj)
 
     def _unwrap_object(self, space, w_value):
         # allow int to pass to char and make sure that str is of length 1
@@ -151,6 +144,9 @@
     c_type      = rffi.UINT
     c_ptrtype   = rffi.UINTP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.uint_w(w_obj))
 
@@ -165,6 +161,9 @@
     c_type      = rffi.LONG
     c_ptrtype   = rffi.LONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.int_w(w_obj)
 
@@ -181,6 +180,9 @@
     c_type      = rffi.ULONG
     c_ptrtype   = rffi.ULONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.uint_w(w_obj)
 
@@ -195,6 +197,9 @@
     c_type      = rffi.LONGLONG
     c_ptrtype   = rffi.LONGLONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.r_longlong_w(w_obj)
 
@@ -209,6 +214,9 @@
     c_type      = rffi.ULONGLONG
     c_ptrtype   = rffi.ULONGLONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.r_ulonglong_w(w_obj)
 
@@ -240,9 +248,11 @@
 
     c_type      = rffi.DOUBLE
     c_ptrtype   = rffi.DOUBLEP
-    wrapper     = 'newfloat'
     typecode    = 'd'
 
+    def _wrap_object(self, space, obj):
+        return space.newfloat(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.float_w(w_obj)
 
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -1,7 +1,7 @@
 import py, os, sys
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib.objectmodel import specialize, instantiate
-from rpython.rlib import rarithmetic, jit
+from rpython.rlib import rarithmetic, rbigint, jit
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper import llinterp
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
@@ -158,11 +158,20 @@
 
     @specialize.argtype(1)
     def newint(self, obj):
-        return FakeInt(rarithmetic.intmask(obj))
+        if not isinstance(obj, int):
+            return FakeLong(rbigint.rbigint.fromrarith_int(obj))
+        return FakeInt(obj)
 
     @specialize.argtype(1)
     def newlong(self, obj):
-        return FakeLong(rarithmetic.intmask(obj))
+        return FakeLong(rbigint.rbigint.fromint(obj))
+
+    @specialize.argtype(1)
+    def newlong_from_rarith_int(self, obj):
+        return FakeLong(rbigint.rbigint.fromrarith_int(obj))
+
+    def newlong_from_rbigint(self, val):
+        return FakeLong(obj)
 
     @specialize.argtype(1)
     def newfloat(self, obj):
@@ -202,10 +211,8 @@
         return w_obj.val
 
     def uint_w(self, w_obj):
-        if isinstance(w_obj, FakeInt):
-            return rarithmetic.r_uint(w_obj.val)
         assert isinstance(w_obj, FakeLong)
-        return rarithmetic.r_uint(w_obj.val)
+        return rarithmetic.r_uint(w_obj.val.touint())
 
     def str_w(self, w_obj):
         assert isinstance(w_obj, FakeString)

From pypy.commits at gmail.com  Wed Feb 15 02:31:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:31:54 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: test and fix
Message-ID: <58a403ea.4a371c0a.12ff0.56d6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90135:292e20099929
Date: 2017-02-15 08:31 +0100
http://bitbucket.org/pypy/pypy/changeset/292e20099929/

Log:	test and fix

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -165,6 +165,8 @@
                 w_value = self.space.newint(intmask(value))
             else:
                 w_value = self.space.newint(value)
+        elif isinstance(value, bool):
+            w_value = self.space.newbool(value)
         elif isinstance(value, int):
             w_value = self.space.newint(value)
         elif isinstance(value, float):
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -283,6 +283,7 @@
         assert pack(">?", False) == '\x00'
         assert pack("@?", True) == '\x01'
         assert pack("@?", False) == '\x00'
+        assert self.struct.unpack("?", 'X')[0] is True
 
     def test_transitiveness(self):
         c = 'a'

From pypy.commits at gmail.com  Wed Feb 15 02:32:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:32:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge f8751e58e3a8
Message-ID: <58a40427.e4361c0a.ddb7.5733@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90136:9f5282f3a6a8
Date: 2017-02-15 08:32 +0100
http://bitbucket.org/pypy/pypy/changeset/9f5282f3a6a8/

Log:	hg merge f8751e58e3a8

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -293,7 +293,7 @@
             return w_result
         if space.isinstance_w(w_result, space.w_int):
             tp = space.type(w_result).name
-            space.warn(space.wrap(
+            space.warn(space.newtext(
                 "__int__ returned non-int (type %s).  "
                 "The ability to return an instance of a strict subclass of int "
                 "is deprecated, and may be removed in a future version of "
@@ -488,13 +488,13 @@
         else:
             name = importname
 
-        mod = Module(self, self.wrap(name))
+        mod = Module(self, self.newtext(name))
         mod.install()
 
         return name
 
     def getbuiltinmodule(self, name, force_init=False, reuse=True):
-        w_name = self.wrap(name)
+        w_name = self.newtext(name)
         w_modules = self.sys.get('modules')
         if not force_init:
             assert reuse
@@ -520,7 +520,7 @@
             if not reuse and w_mod.startup_called:
                 # create a copy of the module.  (see issue1514) eventlet
                 # patcher relies on this behaviour.
-                w_mod2 = self.wrap(Module(self, w_name))
+                w_mod2 = Module(self, w_name)
                 self.setitem(w_modules, w_name, w_mod2)
                 w_mod.getdict(self)  # unlazy w_initialdict
                 self.call_method(w_mod2.getdict(self), 'update',
@@ -566,26 +566,26 @@
         "NOT_RPYTHON: only for initializing the space."
 
         from pypy.module.exceptions import Module
-        w_name = self.wrap('__exceptions__')
+        w_name = self.newtext('__exceptions__')
         self.exceptions_module = Module(self, w_name)
         self.exceptions_module.install()
 
         from pypy.module.imp import Module
-        w_name = self.wrap('_imp')
+        w_name = self.newtext('_imp')
         mod = Module(self, w_name)
         mod.install()
 
         from pypy.module.sys import Module
-        w_name = self.wrap('sys')
+        w_name = self.newtext('sys')
         self.sys = Module(self, w_name)
         self.sys.install()
 
         from pypy.module.__builtin__ import Module
-        w_name = self.wrap('builtins')
+        w_name = self.newtext('builtins')
         self.builtin = Module(self, w_name)
-        w_builtin = self.wrap(self.builtin)
+        w_builtin = self.builtin
         w_builtin.install()
-        self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)
+        self.setitem(self.builtin.w_dict, self.newtext('__builtins__'), w_builtin)
 
         # exceptions was bootstrapped as '__exceptions__' but still
         # lives in pypy/module/exceptions, we rename it below for
@@ -599,7 +599,7 @@
         types_w = (self.get_builtin_types().items() +
                    exception_types_w.items())
         for name, w_type in types_w:
-            self.setitem(self.builtin.w_dict, self.wrap(name), w_type)
+            self.setitem(self.builtin.w_dict, self.newtext(name), w_type)
 
         # install mixed modules
         for mixedname in self.get_builtinmodule_to_install():
@@ -610,10 +610,10 @@
         installed_builtin_modules.append('__exceptions__')
         installed_builtin_modules.sort()
         w_builtin_module_names = self.newtuple(
-            [self.wrap(fn) for fn in installed_builtin_modules])
+            [self.newtext(fn) for fn in installed_builtin_modules])
 
         # force this value into the dict without unlazyfying everything
-        self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'),
+        self.setitem(self.sys.w_dict, self.newtext('builtin_module_names'),
                      w_builtin_module_names)
 
     def get_builtin_types(self):
@@ -741,7 +741,7 @@
     # may also override specific functions for performance.
 
     def not_(self, w_obj):
-        return self.wrap(not self.is_true(w_obj))
+        return self.newbool(not self.is_true(w_obj))
 
     def eq_w(self, w_obj1, w_obj2):
         """Implements equality with the double check 'x is y or x == y'."""
@@ -767,7 +767,7 @@
         w_result = w_obj.immutable_unique_id(self)
         if w_result is None:
             # in the common case, returns an unsigned value
-            w_result = self.wrap(r_uint(compute_unique_id(w_obj)))
+            w_result = self.newint(r_uint(compute_unique_id(w_obj)))
         return w_result
 
     def hash_w(self, w_obj):
@@ -783,10 +783,10 @@
         return self.is_true(self.contains(w_container, w_item))
 
     def setitem_str(self, w_obj, key, w_value):
-        return self.setitem(w_obj, self.wrap(key), w_value)
+        return self.setitem(w_obj, self.newtext(key), w_value)
 
     def finditem_str(self, w_obj, key):
-        return self.finditem(w_obj, self.wrap(key))
+        return self.finditem(w_obj, self.newtext(key))
 
     def finditem(self, w_obj, w_key):
         try:
@@ -1064,13 +1064,13 @@
         return self.newlist([self.newbytes(s) for s in list_s])
 
     def newlist_unicode(self, list_u):
-        return self.newlist([self.wrap(u) for u in list_u])
+        return self.newlist([self.newunicode(u) for u in list_u])
 
     def newlist_int(self, list_i):
-        return self.newlist([self.wrap(i) for i in list_i])
+        return self.newlist([self.newint(i) for i in list_i])
 
     def newlist_float(self, list_f):
-        return self.newlist([self.wrap(f) for f in list_f])
+        return self.newlist([self.newfloat(f) for f in list_f])
 
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
@@ -1166,7 +1166,7 @@
         return w_res
 
     def call_method(self, w_obj, methname, *arg_w):
-        w_meth = self.getattr(w_obj, self.wrap(methname))
+        w_meth = self.getattr(w_obj, self.newtext(methname))
         return self.call_function(w_meth, *arg_w)
 
     def raise_key_error(self, w_key):
@@ -1175,7 +1175,7 @@
 
     def lookup(self, w_obj, name):
         w_type = self.type(w_obj)
-        w_mro = self.getattr(w_type, self.wrap("__mro__"))
+        w_mro = self.getattr(w_type, self.newtext("__mro__"))
         for w_supertype in self.fixedview(w_mro):
             w_value = w_supertype.getdictvalue(self, name)
             if w_value is not None:
@@ -1187,7 +1187,7 @@
         return isinstance(w_obj, GeneratorIterator)
 
     def callable(self, w_obj):
-        return self.wrap(self.lookup(w_obj, "__call__") is not None)
+        return self.newbool(self.lookup(w_obj, "__call__") is not None)
 
     def issequence_w(self, w_obj):
         flag = self.type(w_obj).flag_map_or_seq
@@ -1230,7 +1230,7 @@
 
     def isabstractmethod_w(self, w_obj):
         try:
-            w_result = self.getattr(w_obj, self.wrap("__isabstractmethod__"))
+            w_result = self.getattr(w_obj, self.newtext("__isabstractmethod__"))
         except OperationError as e:
             if e.match(self, self.w_AttributeError):
                 return False
@@ -1283,9 +1283,9 @@
                                          hidden_applevel=hidden_applevel)
         if not isinstance(statement, PyCode):
             raise TypeError('space.exec_(): expected a string, code or PyCode object')
-        w_key = self.wrap('__builtins__')
+        w_key = self.newtext('__builtins__')
         if not self.contains_w(w_globals, w_key):
-            self.setitem(w_globals, w_key, self.wrap(self.builtin))
+            self.setitem(w_globals, w_key, self.builtin)
         return statement.exec_code(self, w_globals, w_locals)
 
     @specialize.arg(2)
@@ -1396,7 +1396,7 @@
                 raise
             if not w_exception:
                 # w_index should be a long object, but can't be sure of that
-                if self.is_true(self.lt(w_index, self.wrap(0))):
+                if self.is_true(self.lt(w_index, self.newint(0))):
                     return -sys.maxint-1
                 else:
                     return sys.maxint
@@ -1820,7 +1820,7 @@
         # with a fileno(), but not an object with an __int__().
         if not self.isinstance_w(w_fd, self.w_int):
             try:
-                w_fileno = self.getattr(w_fd, self.wrap("fileno"))
+                w_fileno = self.getattr(w_fd, self.newtext("fileno"))
             except OperationError as e:
                 if e.match(self, self.w_AttributeError):
                     raise oefmt(self.w_TypeError,
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -444,13 +444,16 @@
             if c in TYPEMAP_PTR_LETTERS:
                 res = func(add_arg, argdesc, rffi.VOIDP)
                 return space.newint(rffi.cast(lltype.Unsigned, res))
+            if c in TYPEMAP_NUMBER_LETTERS:
+                return space.newint(func(add_arg, argdesc, ll_type))
             elif c == 'c':
                 return space.newbytes(func(add_arg, argdesc, ll_type))
+            elif c == 'u':
+                return space.newunicode(func(add_arg, argdesc, ll_type))
             elif c == 'f' or c == 'd' or c == 'g':
                 return space.newfloat(float(func(add_arg, argdesc, ll_type)))
             else:
-                # YYY hard
-                return space.wrap(func(add_arg, argdesc, ll_type))
+                assert 0, "unreachable"
     raise oefmt(space.w_TypeError, "cannot directly read value")
 
 NARROW_INTEGER_TYPES = 'cbhiBIH?'
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1049,15 +1049,20 @@
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
                 return space.newint(item)
-            elif mytype.typecode == 'f':
+            if mytype.typecode in 'ILqQ':
+                return space.newint(item)
+            elif mytype.typecode in 'fd':
                 item = float(item)
                 return space.newfloat(item)
+            elif mytype.typecode == 'c':
+                return space.newbytes(item)
             elif mytype.typecode == 'u':
                 if ord(item) >= 0x110000:
                     raise oefmt(space.w_ValueError,
                                 "array contains a unicode character out of "
                                 "range(0x110000)")
-            return space.wrap(item) # YYY
+                return space.newunicode(item)
+            assert 0, "unreachable"
 
         # interface
 
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -15,7 +15,9 @@
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
-        w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
+        # the w_self argument here is a dummy, the only thing done with w_obj
+        # is call space.type on it
+        w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.w_None)
         space.appexec([space.type(w_obj)], """(methodtype):
             from pickle import Pickler
             Pickler.dispatch[methodtype] = Pickler.save_global
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -264,7 +264,7 @@
     w_dtypes = space.newlist(dtypes_w)
     w_doc = rffi.charp2str(doc)
     w_name = rffi.charp2str(name)
-    w_identity = space.wrap(identity)
+    w_identity = space.newint(identity)
     ufunc_generic = ufuncs.frompyfunc(space, w_funcs, nin, nout, w_dtypes,
                  w_signature, w_identity, w_name, w_doc, stack_inputs=True)
     return ufunc_generic
diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py
--- a/pypy/module/cpyext/structmember.py
+++ b/pypy/module/cpyext/structmember.py
@@ -73,7 +73,7 @@
         w_result = PyUnicode_FromString(space, result)
     elif member_type == T_CHAR:
         result = rffi.cast(rffi.CCHARP, addr)
-        w_result = space.wrap(result[0])
+        w_result = space.newtext(result[0])
     elif member_type == T_OBJECT:
         obj_ptr = rffi.cast(PyObjectP, addr)
         if obj_ptr[0]:
diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -18,7 +18,7 @@
     def call_method(self, space, path, name, args):
         w_method = getattr(self, 'w_' + name)
         if w_method is None:
-            w_method = space.appexec([space.wrap(path), space.wrap(name)],
+            w_method = space.appexec([space.newtext(path), space.newtext(name)],
                 "(path, name): return getattr(__import__(path, fromlist=[name]), name)")
             setattr(self, 'w_' + name, w_method)
         return space.call_args(w_method, args)
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -602,7 +602,8 @@
 
 class W_CharacterBox(W_FlexibleBox):
     def convert_to(self, space, dtype):
-        return dtype.coerce(space, space.wrap(self.raw_str()))
+        # XXX should be newbytes?
+        return dtype.coerce(space, space.newtext(self.raw_str()))
 
     def descr_len(self, space):
         return space.len(self.item(space))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -411,9 +411,12 @@
     def newdict(self, module=True):
         return DictObject({})
 
+    @specialize.argtype(1)
     def newint(self, i):
         if isinstance(i, IntObject):
             return i
+        if isinstance(i, base_int):
+            return LongObject(i)
         return IntObject(i)
 
     def setitem(self, obj, index, value):
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,7 @@
                 val = dtype.itemtype.default_fromstring(space)
             else:
                 try:
-                    val = dtype.coerce(space, space.wrap(piece))
+                    val = dtype.coerce(space, space.newtext(piece))
                 except OperationError as e:
                     if not e.match(space, space.w_ValueError):
                         raise
@@ -476,7 +476,7 @@
                     while not gotit and len(piece) > 0:
                         piece = piece[:-1]
                         try:
-                            val = dtype.coerce(space, space.wrap(piece))
+                            val = dtype.coerce(space, space.newtext(piece))
                             gotit = True
                         except OperationError as e:
                             if not e.match(space, space.w_ValueError):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1263,7 +1263,7 @@
                             op_name, self.get_dtype().get_name())
             shape = self.get_shape()
             if space.is_none(w_axis) or len(shape) <= 1:
-                return space.wrap(getattr(loop, op_name_flat)(self))
+                return space.newint(getattr(loop, op_name_flat)(self))
             else:
                 axis = space.int_w(w_axis)
                 assert axis >= 0
@@ -1340,7 +1340,7 @@
         assert isinstance(multiarray, MixedModule)
         reconstruct = multiarray.get("_reconstruct")
         parameters = space.newtuple([self.getclass(space), space.newtuple(
-            [space.newint(0)]), space.wrap("b")])
+            [space.newint(0)]), space.newtext("b")])
 
         builder = StringBuilder()
         if self.get_dtype().is_object():
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -462,7 +462,7 @@
     signed = True
 
     def to_builtin_type(self, space, box):
-        return space.wrap(self.for_computation(self.unbox(box)))
+        return space.newint(self.for_computation(self.unbox(box)))
 
     def _base_coerce(self, space, w_item):
         if w_item is None:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1206,7 +1206,7 @@
     nargs = interp_attrproperty("nargs", cls=W_Ufunc,
         wrapfn="newint"),
     signature = interp_attrproperty("signature", cls=W_Ufunc,
-        wrapfn="newtext"),
+        wrapfn="newtext_or_none"),
 
     reduce = interp2app(W_Ufunc.descr_reduce),
     outer = interp2app(W_Ufunc.descr_outer),
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -147,10 +147,17 @@
 
     @specialize.argtype(1)
     def appendobj(self, value):
-        if isinstance(value, str):
-            self.result_w.append(self.space.newbytes(value))
+        if isinstance(value, float):
+            w_value = self.space.newfloat(value)
+        elif isinstance(value, str):
+            w_value = self.space.newbytes(value)
+        elif isinstance(value, unicode):
+            w_value = self.space.newunicode(value)
+        elif isinstance(value, bool):
+            w_value = self.space.newbool(value)
         else:
-            self.result_w.append(self.space.wrap(value)) # YYY
+            w_value = self.space.newint(value)
+        self.result_w.append(w_value)
 
     def get_pos(self):
         return self.pos
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
--- a/pypy/module/sys/currentframes.py
+++ b/pypy/module/sys/currentframes.py
@@ -64,7 +64,7 @@
         w_topframe = space.w_None
         w_prevframe = None
         for f in frames:
-            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            w_nextframe = space.call_function(w_fake_frame, space.wrap_none(f))
             if w_prevframe is None:
                 w_topframe = w_nextframe
             else:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -178,6 +178,7 @@
         W_SliceObject(w_start, w_end, w_step)
         return w_some_obj()
 
+    @specialize.argtype(1)
     def newint(self, x):
         return w_some_obj()
 

From pypy.commits at gmail.com  Wed Feb 15 02:36:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:36:33 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix comment: one of us is confused
Message-ID: <58a40501.6d97df0a.2876f.2f8d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90137:e2d28a5f4464
Date: 2017-02-15 08:35 +0100
http://bitbucket.org/pypy/pypy/changeset/e2d28a5f4464/

Log:	fix comment: one of us is confused

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -803,11 +803,13 @@
         return self.is_true(self.contains(w_container, w_item))
 
     def setitem_str(self, w_obj, key, w_value):
-        # key is a "text" (ie str in python2 and unicode in python3)
+        # key is a "text", i.e. a byte string (in python3 it
+        # represents a utf-8-encoded unicode)
         return self.setitem(w_obj, self.newtext(key), w_value)
 
     def finditem_str(self, w_obj, key):
-        # key is a "text" (ie str in python2 and unicode in python3)
+        # key is a "text", i.e. a byte string (in python3 it
+        # represents a utf-8-encoded unicode)
         return self.finditem(w_obj, self.newtext(key))
 
     def finditem(self, w_obj, w_key):

From pypy.commits at gmail.com  Wed Feb 15 02:44:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:44:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 5f267995aab4 (keeping
 wrap and wrap_not_rpython separate)
Message-ID: <58a406d9.2a99df0a.b9a4f.3606@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90138:a13e864fa592
Date: 2017-02-15 08:41 +0100
http://bitbucket.org/pypy/pypy/changeset/a13e864fa592/

Log:	hg merge 5f267995aab4 (keeping wrap and wrap_not_rpython separate)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -783,9 +783,13 @@
         return self.is_true(self.contains(w_container, w_item))
 
     def setitem_str(self, w_obj, key, w_value):
+        # key is a "text", i.e. a byte string (in python3 it
+        # represents a utf-8-encoded unicode)
         return self.setitem(w_obj, self.newtext(key), w_value)
 
     def finditem_str(self, w_obj, key):
+        # key is a "text", i.e. a byte string (in python3 it
+        # represents a utf-8-encoded unicode)
         return self.finditem(w_obj, self.newtext(key))
 
     def finditem(self, w_obj, w_key):
@@ -829,13 +833,14 @@
         return w_u1
 
     def new_interned_str(self, s):
-        """Assumes an identifier (utf-8 encoded str)"""
+        # Assumes an identifier (utf-8 encoded str)
+        # returns a "text" object (ie str in python2 and unicode in python3)
         if not we_are_translated():
             assert type(s) is str
         u = s.decode('utf-8')
         w_s1 = self.interned_strings.get(u)
         if w_s1 is None:
-            w_s1 = self.wrap(u)
+            w_s1 = self.newtext(u)
             self.interned_strings.set(u, w_s1)
         return w_s1
 
@@ -1291,7 +1296,7 @@
     @specialize.arg(2)
     def appexec(self, posargs_w, source):
         """ return value from executing given source at applevel.
-            EXPERIMENTAL. The source must look like
+            The source must look like
                '''(x, y):
                        do_stuff...
                        return result
diff --git a/pypy/module/cppyy/__init__.py b/pypy/module/cppyy/__init__.py
--- a/pypy/module/cppyy/__init__.py
+++ b/pypy/module/cppyy/__init__.py
@@ -40,4 +40,4 @@
         from pypy.module.cppyy import capi
         capi.verify_backend(space)      # may raise ImportError
 
-        space.call_method(space.wrap(self), '_init_pythonify')
+        space.call_method(self, '_init_pythonify')
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -972,4 +972,5 @@
     Unicode strings.  CRLF is considered to be one line break.  If
     keepend is 0, the Line break characters are not included in the
     resulting strings."""
-    return space.call_method(w_str, "splitlines", space.newbool(bool(keepend)))
+    w_keepend = space.newbool(bool(rffi.cast(lltype.Signed, keepend)))
+    return space.call_method(w_str, "splitlines", w_keepend)
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -860,7 +860,7 @@
             alignment = -1
         format = dtype_from_spec(space, obj[0], alignment=alignment)
         if len(obj) > 2:
-            title = space.wrap(obj[2])
+            title = obj[2]
         else:
             title = space.w_None
         allfields.append((fname_w, format, num, title))
@@ -874,7 +874,7 @@
     else:
         alignment = -1
     for i in range(len(names)):
-        aslist.append(space.newtuple([space.wrap(names[i]), space.wrap(formats[i])]))
+        aslist.append(space.newtuple([names[i], formats[i]]))
     return dtype_from_list(space, space.newlist(aslist), False, alignment, offsets=offsets)
     
 def dtype_from_dict(space, w_dict, alignment):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -12,6 +12,7 @@
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rarithmetic import base_int, widen, is_valid_int
 from rpython.rlib.objectmodel import import_from_mixin, we_are_translated
+from rpython.rlib.objectmodel import not_rpython
 from rpython.rlib import jit
 
 # Object imports
@@ -41,8 +42,9 @@
     library in Restricted Python."""
     import_from_mixin(DescrOperation)
 
+    @not_rpython
     def initialize(self):
-        """NOT_RPYTHON: only for initializing the space
+        """only for initializing the space
 
         Setup all the object types and implementations.
         """
@@ -130,15 +132,11 @@
         assert typedef is not None
         return self.fromcache(TypeCache).getorbuild(typedef)
 
-    @specialize.argtype(1)
+    @not_rpython # only for tests
     def wrap(self, x):
-        "Wraps the Python value 'x' into one of the wrapper classes."
-        # You might notice that this function is rather conspicuously
-        # not RPython.  We can get away with this because the function
-        # is specialized (see after the function body).  Also worth
-        # noting is that the isinstance's involving integer types
-        # behave rather differently to how you might expect during
-        # annotation (see pypy/annotation/builtin.py)
+        """ Wraps the Python value 'x' into one of the wrapper classes. This
+        should only be used for tests, in real code you need to use the
+        explicit new* methods."""
         if x is None:
             return self.w_None
         if isinstance(x, OperationError):
@@ -236,15 +234,15 @@
             self.wrap("refusing to wrap cpython value %r" % (x,))
         )
 
+    @not_rpython
     def wrap_exception_cls(self, x):
-        """NOT_RPYTHON"""
         if hasattr(self, 'w_' + x.__name__):
             w_result = getattr(self, 'w_' + x.__name__)
             return w_result
         return None
 
+    @not_rpython
     def wraplong(self, x):
-        "NOT_RPYTHON"
         if self.config.objspace.std.withsmalllong:
             from rpython.rlib.rarithmetic import r_longlong
             try:
@@ -257,8 +255,8 @@
                 return W_SmallLongObject(rx)
         return W_LongObject.fromlong(x)
 
+    @not_rpython
     def unwrap(self, w_obj):
-        """NOT_RPYTHON"""
         # _____ this code is here to support testing only _____
         if isinstance(w_obj, W_Root):
             return w_obj.unwrap(self)
@@ -579,7 +577,7 @@
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
             return (a, b, c)
-        w_indices = self.getattr(w_slice, self.wrap('indices'))
+        w_indices = self.getattr(w_slice, self.newbytes('indices'))
         w_tup = self.call_function(w_indices, w_length)
         l_w = self.unpackiterable(w_tup)
         if not len(l_w) == 3:
@@ -679,7 +677,7 @@
                 not w_obj.user_overridden_class):
             w_obj.setitem_str(key, w_value)
         else:
-            self.setitem(w_obj, self.wrap(key), w_value)
+            self.setitem(w_obj, self.newtext(key), w_value)
 
     def getindex_w(self, w_obj, w_exception, objdescr=None):
         if type(w_obj) is W_IntObject:

From pypy.commits at gmail.com  Wed Feb 15 02:44:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Feb 2017 23:44:27 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: oops, fix
Message-ID: <58a406db.2596df0a.cdc97.7396@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90139:b1e1f945f231
Date: 2017-02-15 08:43 +0100
http://bitbucket.org/pypy/pypy/changeset/b1e1f945f231/

Log:	oops, fix

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -840,7 +840,7 @@
         u = s.decode('utf-8')
         w_s1 = self.interned_strings.get(u)
         if w_s1 is None:
-            w_s1 = self.newtext(u)
+            w_s1 = self.newunicode(u)
             self.interned_strings.set(u, w_s1)
         return w_s1
 

From pypy.commits at gmail.com  Wed Feb 15 04:06:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 01:06:49 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: here we process bytes
Message-ID: <58a41a29.4a6b1c0a.f4755.6f15@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90140:31f7ee8dcadc
Date: 2017-02-15 09:01 +0100
http://bitbucket.org/pypy/pypy/changeset/31f7ee8dcadc/

Log:	here we process bytes

diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -103,7 +103,7 @@
     def read(self, n):
         space = self.space
         w_ret = space.call_function(self.func, space.newint(n))
-        ret = space.text_w(w_ret)
+        ret = space.bytes_w(w_ret)
         if len(ret) != n:
             self.raise_eof()
         return ret

From pypy.commits at gmail.com  Wed Feb 15 04:06:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 01:06:54 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: two places where newtext is
 better than newbytes
Message-ID: <58a41a2e.2da9df0a.fcf9b.4a73@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90142:19dc3e9fb603
Date: 2017-02-15 09:36 +0100
http://bitbucket.org/pypy/pypy/changeset/19dc3e9fb603/

Log:	two places where newtext is better than newbytes

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -153,7 +153,7 @@
             else:
                 return self.newint(x)
         if isinstance(x, str):
-            return self.newbytes(x)
+            return self.newtext(x)
         if isinstance(x, unicode):
             return self.newunicode(x)
         if isinstance(x, float):
@@ -547,7 +547,7 @@
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
             return (a, b, c)
-        w_indices = self.getattr(w_slice, self.newbytes('indices'))
+        w_indices = self.getattr(w_slice, self.newtext('indices'))
         w_tup = self.call_function(w_indices, w_length)
         l_w = self.unpackiterable(w_tup)
         if not len(l_w) == 3:

From pypy.commits at gmail.com  Wed Feb 15 04:06:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 01:06:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 67d4325b8871 (but broken)
Message-ID: <58a41a30.0a8edf0a.a0288.4673@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90143:b1f56f360399
Date: 2017-02-15 10:01 +0100
http://bitbucket.org/pypy/pypy/changeset/b1f56f360399/

Log:	hg merge 67d4325b8871 (but broken)

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -209,15 +209,15 @@
                                    consts.CO_COROUTINE |
                                    consts.CO_ITERABLE_COROUTINE):
             e2 = OperationError(space.w_RuntimeError,
-                                space.wrap("%s raised StopIteration" %
-                                           self.KIND))
+                                space.newtext("%s raised StopIteration" %
+                                              self.KIND))
             e2.chain_exceptions(space, e)
             e2.set_cause(space, e.get_w_value(space))
             e2.record_context(space, space.getexecutioncontext())
             raise e2
         else:
-            space.warn(space.wrap(u"generator '%s' raised StopIteration"
-                                  % self.get_qualname()),
+            space.warn(space.newunicode(u"generator '%s' raised StopIteration"
+                                        % self.get_qualname()),
                        space.w_PendingDeprecationWarning)
 
     def descr_throw(self, w_type, w_val=None, w_tb=None):
@@ -251,7 +251,7 @@
 
         if tb is None:
             tb = space.getattr(operr.get_w_value(space),
-                               space.wrap('__traceback__'))
+                               space.newtext('__traceback__'))
             if not space.is_w(tb, space.w_None):
                 operr.set_traceback(tb)
         return self.send_error(operr)
@@ -297,7 +297,7 @@
             return space.w_None
 
     def descr__name__(self, space):
-        return space.wrap(self.get_name().decode('utf-8'))
+        return space.newtext(self.get_name())
 
     def descr_set__name__(self, space, w_name):
         if space.isinstance_w(w_name, space.w_unicode):
@@ -307,7 +307,7 @@
                         "__name__ must be set to a string object")
 
     def descr__qualname__(self, space):
-        return space.wrap(self.get_qualname())
+        return space.newunicode(self.get_qualname())
 
     def descr_set__qualname__(self, space, w_name):
         try:
@@ -343,7 +343,7 @@
 
     def descr__iter__(self):
         """Implement iter(self)."""
-        return self.space.wrap(self)
+        return self
 
     def descr_next(self):
         """Implement next(self)."""
@@ -390,7 +390,7 @@
     KIND = "coroutine"
 
     def descr__await__(self, space):
-        return space.wrap(CoroutineWrapper(self))
+        return CoroutineWrapper(self)
 
     def _finalize_(self):
         # If coroutine was never awaited on issue a RuntimeWarning.
@@ -399,7 +399,7 @@
            self.frame.last_instr == -1:
             space = self.space
             msg = u"coroutine '%s' was never awaited" % self.get_qualname()
-            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            space.warn(space.newunicode(msg), space.w_RuntimeWarning)
         GeneratorOrCoroutine._finalize_(self)
 
 
@@ -410,7 +410,7 @@
         self.coroutine = coroutine
 
     def descr__iter__(self, space):
-        return space.wrap(self)
+        return self
 
     def descr__next__(self, space):
         return self.coroutine.send_ex(space.w_None)
@@ -436,10 +436,10 @@
         self.w_aiter = w_aiter
 
     def descr__await__(self, space):
-        return space.wrap(self)
+        return self
 
     def descr__iter__(self, space):
-        return space.wrap(self)
+        return self
 
     def descr__next__(self, space):
         raise OperationError(space.w_StopIteration, self.w_aiter)
@@ -457,7 +457,7 @@
         w_yf.descr_close()
     else:
         try:
-            w_close = space.getattr(w_yf, space.wrap("close"))
+            w_close = space.getattr(w_yf, space.newtext("close"))
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
                 # aaaaaaaah but that's what CPython does too
@@ -470,7 +470,7 @@
     if isinstance(w_inputvalue_or_err, SApplicationException):
         operr = w_inputvalue_or_err.operr
         try:
-            w_meth = space.getattr(w_yf, space.wrap("throw"))
+            w_meth = space.getattr(w_yf, space.newtext("throw"))
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
                 raise
@@ -481,7 +481,7 @@
         operr.normalize_exception(space)
         w_exc = operr.w_type
         w_val = operr.get_w_value(space)
-        w_tb  = space.wrap(operr.get_traceback())
+        w_tb  = operr.get_w_traceback(space)
         return space.call_function(w_meth, w_exc, w_val, w_tb)
     else:
         return space.call_method(w_yf, "send", w_inputvalue_or_err)
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -99,6 +99,8 @@
 
     def str_w(self, s):
         return str(s)
+    def text_w(self, s):
+        return self.str_w(s)
 
     def text_w(self, s):
         return self.str_w(s)
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -184,7 +184,7 @@
         copy.maxlen = self.maxlen
         copy.extend(self.iter())
         copy.extend(deque.iter())
-        return self.space.wrap(copy)
+        return copy
 
     def iadd(self, w_iterable):
         self.extend(w_iterable)
@@ -199,7 +199,7 @@
         for _ in range(num):
             copied.extend(self)
 
-        return space.wrap(copied)
+        return copied
 
     def rmul(self, w_int):
         return self.mul(w_int)
@@ -208,10 +208,10 @@
         space = self.space
         num = space.int_w(w_int)
         if self.len == 0 or num == 1:
-            return space.wrap(self)
+            return self
         if num <= 0:
             self.clear()
-            return space.wrap(self)
+            return self
         # use a copy to extend self
         copy = W_Deque(space)
         copy.maxlen = self.maxlen
@@ -220,7 +220,7 @@
         for _ in range(num - 1):
             self.extend(copy)
 
-        return space.wrap(self)
+        return self
 
     def extendleft(self, w_iterable):
         "Extend the left side of the deque with elements from the iterable"
@@ -369,7 +369,7 @@
                 if i < start:
                     continue
                 if space.eq_w(w_obj, w_x):
-                    return space.wrap(i)
+                    return space.newint(i)
                 self.checklock(lock)
             except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
@@ -642,7 +642,7 @@
         return w_x
 
     def reduce(self):
-        w_i = self.space.wrap(self.deque.len - self.counter)
+        w_i = self.space.newint(self.deque.len - self.counter)
         return self.space.newtuple([self.space.gettypefor(W_DequeIter),
                                     self.space.newtuple([self.deque, w_i])])
 
@@ -710,7 +710,7 @@
         return w_x
 
     def reduce(self):
-        w_i = self.space.wrap(self.deque.len - self.counter)
+        w_i = self.space.newint(self.deque.len - self.counter)
         return self.space.newtuple([self.space.gettypefor(W_DequeRevIter),
                                     self.space.newtuple([self.deque, w_i])])
 
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -180,7 +180,7 @@
 
 # XXX Hack to seperate rpython and pypy
 def ipaddr_from_object(space, w_sockaddr):
-    host = space.str_w(space.getitem(w_sockaddr, space.newint(0)))
+    host = space.text_w(space.getitem(w_sockaddr, space.newint(0)))
     addr = rsocket.makeipaddr(host)
     fill_from_object(addr, space, w_sockaddr)
     return addr
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -408,7 +408,7 @@
             except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
-                return space.wrap(count)
+                return space.newint(count)
             if space.eq_w(w_next, w_item):
                 count += 1
 
@@ -424,7 +424,7 @@
                 raise oefmt(space.w_ValueError,
                             "sequence.index(x): x not in sequence")
             if space.eq_w(w_next, w_item):
-                return space.wrap(index)
+                return space.newint(index)
             index += 1
 
     def hash(space, w_obj):
@@ -483,7 +483,7 @@
             return w_result
         if space.isinstance_w(w_result, space.w_int):
             tp = space.type(w_result).name
-            space.warn(space.wrap(
+            space.warn(space.newtext(
                 "__index__ returned non-int (type %s).  "
                 "The ability to return an instance of a strict subclass of int "
                 "is deprecated, and may be removed in a future version of "
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -92,6 +92,8 @@
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
             self._interplevel_classes[w_type] = cls
+        self.w_bytes = self.w_str
+        self.w_text = self.w_str # this is w_unicode on Py3
         self.w_dict.flag_map_or_seq = 'M'
         self.w_list.flag_map_or_seq = 'S'
         self.w_tuple.flag_map_or_seq = 'S'
@@ -132,7 +134,6 @@
         assert typedef is not None
         return self.fromcache(TypeCache).getorbuild(typedef)
 
-    @not_rpython # only for tests
     def wrap(self, x):
         """ Wraps the Python value 'x' into one of the wrapper classes. This
         should only be used for tests, in real code you need to use the
@@ -153,7 +154,7 @@
             try:
                 unicode_x = x.decode('ascii')
             except UnicodeDecodeError:
-                unicode_x = self._wrap_ascii_replace(x)
+                unicode_x = self._wrap_string_old(x)
             return self.newunicode(unicode_x)
         if isinstance(x, unicode):
             return self.newunicode(x)
@@ -167,24 +168,13 @@
             return self.newint(x)
         return self._wrap_not_rpython(x)
 
-    def _wrap_ascii_replace(self, x):
-        # XXX should disappear soon?
-        # poor man's x.decode('ascii', 'replace'), since it's not
-        # supported by RPython
-        if not we_are_translated():
-            print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
-        lst = []
-        for ch in x:
-            ch = ord(ch)
-            if ch > 127:
-                lst.append(u'\ufffd')
-            else:
-                lst.append(unichr(ch))
-        unicode_x = u''.join(lst)
-        return unicode_x
+    def _wrap_string_old(self, x):
+        # XXX should disappear soon
+        print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
+        return self.newtext(x)
 
+    @not_rpython # only for tests
     def _wrap_not_rpython(self, x):
-        "NOT_RPYTHON"
         # _____ this code is here to support testing only _____
 
         # we might get there in non-translated versions if 'x' is
@@ -577,7 +567,7 @@
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
             return (a, b, c)
-        w_indices = self.getattr(w_slice, self.newbytes('indices'))
+        w_indices = self.getattr(w_slice, self.newtext('indices'))
         w_tup = self.call_function(w_indices, w_length)
         l_w = self.unpackiterable(w_tup)
         if not len(l_w) == 3:

From pypy.commits at gmail.com  Wed Feb 15 04:06:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 01:06:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge ffe62ff416aa
Message-ID: <58a41a2c.45aadf0a.5806.41ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90141:6b32e82270a9
Date: 2017-02-15 09:16 +0100
http://bitbucket.org/pypy/pypy/changeset/6b32e82270a9/

Log:	hg merge ffe62ff416aa

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -83,7 +83,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
             try:
@@ -91,7 +91,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
 
@@ -148,7 +148,7 @@
             if verbose:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
             if must_leave:
@@ -202,7 +202,7 @@
         except OperationError as e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
 
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -97,9 +97,9 @@
             else:
                 try:
                     if use_repr:
-                        exc_value = space.str_w(space.repr(w_value))
+                        exc_value = space.text_w(space.repr(w_value))
                     else:
-                        exc_value = space.str_w(space.str(w_value))
+                        exc_value = space.text_w(space.str(w_value))
                 except OperationError:
                     # oups, cannot __str__ the exception object
                     exc_value = ("<exception %s() failed>" %
@@ -259,7 +259,7 @@
             objrepr = ''
         else:
             try:
-                objrepr = space.str_w(space.repr(w_object))
+                objrepr = space.text_w(space.repr(w_object))
             except OperationError:
                 objrepr = "<object repr() failed>"
         #
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -222,7 +222,7 @@
         if not space.isinstance_w(w_globals, space.w_dict):
             raise oefmt(space.w_TypeError, "expected dict")
         if not space.is_none(w_name):
-            name = space.str_w(w_name)
+            name = space.text_w(w_name)
         else:
             name = None
         if not space.is_none(w_argdefs):
@@ -423,8 +423,8 @@
         return space.newtext(self.name)
 
     def fset_func_name(self, space, w_name):
-        if space.isinstance_w(w_name, space.w_unicode):
-            self.name = space.str_w(w_name)
+        if space.isinstance_w(w_name, space.w_text):
+            self.name = space.text_w(w_name)
         else:
             raise oefmt(space.w_TypeError,
                         "__name__ must be set to a string object")
@@ -559,7 +559,7 @@
 
     def descr_method_getattribute(self, w_attr):
         space = self.space
-        if space.str_w(w_attr) != '__doc__':
+        if space.text_w(w_attr) != '__doc__':
             try:
                 return space.call_method(space.w_object, '__getattribute__',
                                          self, w_attr)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1026,6 +1026,7 @@
     def IMPORT_NAME(self, nameindex, next_instr):
         space = self.space
         w_modulename = self.getname_w(nameindex)
+        modulename = self.space.text_w(w_modulename)
         w_fromlist = self.popvalue()
 
         w_flag = self.popvalue()
@@ -1046,6 +1047,7 @@
             w_locals = d.w_locals
         if w_locals is None:            # CPython does this
             w_locals = space.w_None
+        w_modulename = space.newtext(modulename)
         w_globals = self.get_w_globals()
         if w_flag is None:
             w_obj = space.call_function(w_import, w_modulename, w_globals,
@@ -1076,7 +1078,7 @@
                 raise
             try:
                 w_pkgname = space.getattr(
-                    w_module, space.newunicode(u'__name__'))
+                    w_module, space.newtext('__name__'))
                 w_fullname = space.newunicode(u'%s.%s' %
                     (space.unicode_w(w_pkgname), space.unicode_w(w_name)))
                 return space.getitem(space.sys.get('modules'), w_fullname)
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -46,7 +46,7 @@
             return space.call_function(w_selftype, self.w_starttype, w_obj)
 
     def getattribute(self, space, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         # only use a special logic for bound super objects and not for
         # getting the __class__ of the super object itself.
         if self.w_objtype is not None and name != '__class__':
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -535,7 +535,7 @@
     if w_encoding is None:
         encoding = space.sys.defaultencoding
     else:
-        encoding = space.str_w(w_encoding)
+        encoding = space.text_w(w_encoding)
     w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
     return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
 
@@ -558,7 +558,7 @@
     if w_encoding is None:
         encoding = space.sys.defaultencoding
     else:
-        encoding = space.str_w(w_encoding)
+        encoding = space.text_w(w_encoding)
     w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1))
     return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
 
diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -26,7 +26,7 @@
     if space.is_none(w_locale):
         locale = None
     else:
-        locale = space.str_w(w_locale)
+        locale = space.text_w(w_locale)
     try:
         result = rlocale.setlocale(category, locale)
     except rlocale.LocaleError as e:
@@ -183,7 +183,7 @@
             finally:
                 rffi.free_charp(msg_c)
         else:
-            domain = space.str_w(w_domain)
+            domain = space.text_w(w_domain)
             domain_c = rffi.str2charp(domain)
             msg_c = rffi.str2charp(msg)
             try:
@@ -218,7 +218,7 @@
             finally:
                 rffi.free_charp(msg_c)
         else:
-            domain = space.str_w(w_domain)
+            domain = space.text_w(w_domain)
             domain_c = rffi.str2charp(domain)
             msg_c = rffi.str2charp(msg)
             try:
@@ -246,7 +246,7 @@
             result = _textdomain(domain)
             result = rffi.charp2str(result)
         else:
-            domain = space.str_w(w_domain)
+            domain = space.text_w(w_domain)
             domain_c = rffi.str2charp(domain)
             try:
                 result = _textdomain(domain_c)
@@ -276,7 +276,7 @@
             finally:
                 rffi.free_charp(domain_c)
         else:
-            dir = space.str_w(w_dir)
+            dir = space.text_w(w_dir)
             domain_c = rffi.str2charp(domain)
             dir_c = rffi.str2charp(dir)
             try:
@@ -307,7 +307,7 @@
                 finally:
                     rffi.free_charp(domain_c)
             else:
-                codeset = space.str_w(w_codeset)
+                codeset = space.text_w(w_codeset)
                 domain_c = rffi.str2charp(domain)
                 codeset_c = rffi.str2charp(codeset)
                 try:
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -49,11 +49,11 @@
         return self.w_calls
 
     def repr(self, space):
-        frame_repr = space.str_w(space.repr(self.frame))
+        frame_repr = space.text_w(space.repr(self.frame))
         if not self.w_calls:
             calls_repr = "None"
         else:
-            calls_repr = space.str_w(space.repr(self.w_calls))
+            calls_repr = space.text_w(space.repr(self.w_calls))
         return space.newtext('("%s", %d, %d, %f, %f, %s)' % (
             frame_repr, self.callcount, self.reccallcount,
             self.tt, self.it, calls_repr))
@@ -85,7 +85,7 @@
         self.tt = tt
 
     def repr(self, space):
-        frame_repr = space.str_w(space.repr(self.frame))
+        frame_repr = space.text_w(space.repr(self.frame))
         return space.newtext('("%s", %d, %d, %f, %f)' % (
             frame_repr, self.callcount, self.reccallcount, self.tt, self.it))
 
@@ -235,7 +235,7 @@
     # This class should not be seen at app-level, but is useful to
     # contain a (w_func, w_type) pair returned by prepare_spec().
     # Turning this pair into a string cannot be done eagerly in
-    # an @elidable function because of space.str_w(), but it can
+    # an @elidable function because of space.text_w(), but it can
     # be done lazily when we really want it.
 
     _immutable_fields_ = ['w_func', 'w_type']
@@ -253,7 +253,7 @@
                 s = create_spec_for_object(space, self.w_type)
             else:
                 s = create_spec_for_method(space, self.w_func, self.w_type)
-            self.w_string = space.newtext(s)
+            self.w_string = space.newunicode(s)
         return self.w_string
 
 W_DelayedBuiltinStr.typedef = TypeDef(
diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py
--- a/pypy/module/_minimal_curses/interp_curses.py
+++ b/pypy/module/_minimal_curses/interp_curses.py
@@ -51,7 +51,7 @@
         if space.is_none(w_termname):
             _curses_setupterm_null(fd)
         else:
-            _curses_setupterm(space.str_w(w_termname), fd)
+            _curses_setupterm(space.text_w(w_termname), fd)
     except curses_error as e:
         raise convert_error(space, e)
 
diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -33,7 +33,7 @@
         return space.newtext(self.errors)
 
     def fset_errors(self, space, w_errors):
-        self.errors = space.str_w(w_errors)
+        self.errors = space.text_w(w_errors)
 
 
 class MultibyteIncrementalDecoder(MultibyteIncrementalBase):
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -786,7 +786,7 @@
             args_w = [self.args_w[0], w_tuple]
             args_repr = space.unicode_w(space.repr(space.newtuple(args_w)))
             clsname = self.getclass(space).getname(space)
-            return space.newtext(clsname + args_repr)
+            return space.newunicode(clsname + args_repr)
         else:
             return W_Exception.descr_repr(self, space)
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -380,7 +380,7 @@
     module object.
     """
     log_pyverbose(space, 1, "import %s # compiled from %s\n" %
-                  (space.str_w(w_modulename), cpathname))
+                  (space.text_w(w_modulename), cpathname))
 
     if magic != get_pyc_magic(space):
         raise oefmt(space.w_ImportError, "Bad magic number in %s", cpathname)
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -51,7 +51,7 @@
     w_modulename = space.getattr(w_spec, space.newtext("name"))
     w_path = space.getattr(w_spec, space.newtext("origin"))
     filename = space.fsencode_w(w_path)
-    importing.load_c_extension(space, filename, space.str_w(w_modulename))
+    importing.load_c_extension(space, filename, space.text_w(w_modulename))
     return importing.check_sys_modules(space, w_modulename)
 
 def create_builtin(space, w_spec):
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -27,11 +27,11 @@
 
     def repr_w(self):
         space = self.space
-        c = space.str_w(space.repr(self.w_c))
+        c = space.text_w(space.repr(self.w_c))
         if self.single_argument():
             s = 'count(%s)' % (c,)
         else:
-            step = space.str_w(space.repr(self.w_step))
+            step = space.text_w(space.repr(self.w_step))
             s = 'count(%s, %s)' % (c, step)
         return self.space.newtext(s)
 
@@ -107,7 +107,7 @@
         return self.space.newint(self.count)
 
     def repr_w(self):
-        objrepr = self.space.str_w(self.space.repr(self.w_obj))
+        objrepr = self.space.text_w(self.space.repr(self.w_obj))
         if self.counting:
             s = 'repeat(%s, %d)' % (objrepr, self.count)
         else:
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -87,7 +87,7 @@
     def read(self, n):
         space = self.space
         w_ret = space.call_function(self.func, space.newint(n))
-        ret = space.str_w(w_ret)
+        ret = space.bytes_w(w_ret)
         if len(ret) < n:
             self.raise_eof()
         if len(ret) > n:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1954,7 +1954,7 @@
     # XXX slightly non-nice, reuses the sysconf of the underlying os module
     if space.isinstance_w(w_name, space.w_unicode):
         try:
-            num = namespace[space.str_w(w_name)]
+            num = namespace[space.text_w(w_name)]
         except KeyError:
             raise oefmt(space.w_ValueError, "unrecognized configuration name")
     else:
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -36,7 +36,7 @@
         space = self.space
         for name in interp_pyexpat.xml_model_list:
             value = getattr(interp_pyexpat, name)
-            space.setattr(self, space.newtext(name), space.newtext(value))
+            space.setattr(self, space.newtext(name), space.wrap(value))
 
 class Module(MixedModule):
     "Python wrapper for Expat parser."
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -679,12 +679,12 @@
         if space.is_w(w_context, space.w_None):
             context = None
         else:
-            context = space.str_w(w_context)
+            context = space.text_w(w_context)
 
         if space.is_none(w_encoding):
             encoding = None
         else:
-            encoding = space.str_w(w_encoding)
+            encoding = space.text_w(w_encoding)
 
         xmlparser = XML_ExternalEntityParserCreate(
             self.itself, context, encoding)
@@ -816,8 +816,8 @@
 Return a new XML parser object."""
     if space.is_none(w_encoding):
         encoding = None
-    elif space.isinstance_w(w_encoding, space.w_unicode):
-        encoding = space.str_w(w_encoding)
+    elif space.isinstance_w(w_encoding, space.w_text):
+        encoding = space.text_w(w_encoding)
     else:
         raise oefmt(space.w_TypeError,
                     "ParserCreate() argument 1 must be str or None, not %T",
@@ -825,8 +825,8 @@
 
     if space.is_none(w_namespace_separator):
         namespace_separator = 0
-    elif space.isinstance_w(w_namespace_separator, space.w_unicode):
-        separator = space.str_w(w_namespace_separator)
+    elif space.isinstance_w(w_namespace_separator, space.w_text):
+        separator = space.text_w(w_namespace_separator)
         if len(separator) == 0:
             namespace_separator = 0
         elif len(separator) == 1:
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -299,7 +299,7 @@
         for compiled, _, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
                 w_source = self.get_data(space, filename + ext)
-                source = space.str_w(w_source)
+                source = space.bytes_w(w_source)
                 if compiled:
                     magic = importing._get_long(source[:4])
                     timestamp = importing._get_long(source[4:8])
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -65,7 +65,7 @@
 
 def get_attribute_name(space, w_obj, w_name):
     try:
-        return space.str_w(w_name)
+        return space.text_w(w_name)
     except OperationError as e:
         if e.match(space, space.w_UnicodeEncodeError):
             raiseattrerror(space, w_obj, w_name)
diff --git a/pypy/objspace/std/callmethod.py b/pypy/objspace/std/callmethod.py
--- a/pypy/objspace/std/callmethod.py
+++ b/pypy/objspace/std/callmethod.py
@@ -44,7 +44,7 @@
 
     w_type = space.type(w_obj)
     if w_type.has_object_getattribute():
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         # bit of a mess to use these internal functions, but it allows the
         # mapdict caching below to work without an additional lookup
         version_tag = w_type.version_tag()
@@ -106,7 +106,7 @@
                 break
             w_value = f.popvalue()
             w_key = f.popvalue()
-            key = f.space.str_w(w_key)
+            key = f.space.text_w(w_key)
             keywords[n_kwargs] = key
             keywords_w[n_kwargs] = w_value
 
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -740,7 +740,7 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_unicode):
+        if space.is_w(w_lookup_type, space.w_text):
             return self.getitem_str(w_dict, space.text_w(w_key))
         elif _never_equal_to_string(space, w_lookup_type):
             return None
@@ -759,7 +759,7 @@
 
     def setitem(self, w_dict, w_key, w_value):
         space = self.space
-        if space.is_w(space.type(w_key), space.w_unicode):
+        if space.is_w(space.type(w_key), space.w_text):
             self.setitem_str(w_dict, self.space.text_w(w_key), w_value)
         else:
             self.switch_to_object_strategy(w_dict)
@@ -767,7 +767,7 @@
 
     def setdefault(self, w_dict, w_key, w_default):
         space = self.space
-        if space.is_w(space.type(w_key), space.w_unicode):
+        if space.is_w(space.type(w_key), space.w_text):
             key = space.text_w(w_key)
             w_result = self.getitem_str(w_dict, key)
             if w_result is not None:
@@ -782,7 +782,7 @@
         space = self.space
         w_key_type = space.type(w_key)
         w_obj = self.unerase(w_dict.dstorage)
-        if space.is_w(w_key_type, space.w_unicode):
+        if space.is_w(w_key_type, space.w_text):
             key = self.space.text_w(w_key)
             flag = w_obj.deldictvalue(space, key)
             if not flag:
@@ -983,7 +983,7 @@
             return space._handle_getattribute(w_descr, w_obj, w_name)
         version_tag = w_type.version_tag()
         if version_tag is not None:
-            name = space.str_w(w_name)
+            name = space.text_w(w_name)
             # We need to care for obscure cases in which the w_descr is
             # a MutableCell, which may change without changing the version_tag
             _, w_descr = w_type._pure_lookup_where_with_method_cache(
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -106,9 +106,9 @@
 
     def descr_repr(self, space):
         return space.newtext("slice(%s, %s, %s)" % (
-            space.str_w(space.repr(self.w_start)),
-            space.str_w(space.repr(self.w_stop)),
-            space.str_w(space.repr(self.w_step))))
+            space.text_w(space.repr(self.w_start)),
+            space.text_w(space.repr(self.w_stop)),
+            space.text_w(space.repr(self.w_step))))
 
     def descr__reduce__(self, space):
         from pypy.objspace.std.sliceobject import W_SliceObject
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1230,6 +1230,7 @@
             return unicode
         return type(w_obj)
     w_unicode = unicode
+    w_text = unicode
     w_bytes = str
 
     def text_w(self, u):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -663,7 +663,7 @@
             return space.newtext("<class '%s'>" % (self.name,))
 
     def descr_getattribute(self, space, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         w_descr = space.lookup(self, name)
         if w_descr is not None:
             if space.is_data_descr(w_descr):
@@ -724,14 +724,14 @@
             return space.call_function(newfunc, w_winner, w_name, w_bases, w_dict)
         w_typetype = w_winner
 
-    name = space.str_w(w_name)
+    name = space.text_w(w_name)
     assert isinstance(name, str)
     if '\x00' in name:
         raise oefmt(space.w_ValueError, "type name must not contain null characters")
     dict_w = {}
     dictkeys_w = space.listview(w_dict)
     for w_key in dictkeys_w:
-        key = space.str_w(w_key)
+        key = space.text_w(w_key)
         dict_w[key] = space.getitem(w_dict, w_key)
     w_type = space.allocate_instance(W_TypeObject, w_typetype)
     W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object],
@@ -789,7 +789,7 @@
         raise oefmt(space.w_TypeError,
                     "can only assign string to %N.__name__, not '%T'",
                     w_type, w_value)
-    name = space.str_w(w_value)
+    name = space.text_w(w_value)
     if '\x00' in name:
         raise oefmt(space.w_ValueError, "type name must not contain null characters")
     w_type.name = name
@@ -1151,7 +1151,7 @@
     slot_name = mangle(slot_name, w_self.name)
     if slot_name not in w_self.dict_w:
         # Force interning of slot names.
-        slot_name = space.str_w(space.new_interned_str(slot_name))
+        slot_name = space.text_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
         w_self.dict_w[slot_name] = member

From pypy.commits at gmail.com  Wed Feb 15 04:06:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 01:06:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fix merge
Message-ID: <58a41a32.28a8df0a.9df97.489a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90144:9c8c313486ee
Date: 2017-02-15 10:06 +0100
http://bitbucket.org/pypy/pypy/changeset/9c8c313486ee/

Log:	fix merge

diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -102,9 +102,6 @@
     def text_w(self, s):
         return self.str_w(s)
 
-    def text_w(self, s):
-        return self.str_w(s)
-
     def unicode_w(self, s):
         return unicode(s)
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -92,8 +92,6 @@
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
             self._interplevel_classes[w_type] = cls
-        self.w_bytes = self.w_str
-        self.w_text = self.w_str # this is w_unicode on Py3
         self.w_dict.flag_map_or_seq = 'M'
         self.w_list.flag_map_or_seq = 'S'
         self.w_tuple.flag_map_or_seq = 'S'

From pypy.commits at gmail.com  Wed Feb 15 04:36:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 01:36:08 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: Fix test for py3.5
Message-ID: <58a42108.28a8df0a.9df97.50f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90145:da67900609d3
Date: 2017-02-15 10:35 +0100
http://bitbucket.org/pypy/pypy/changeset/da67900609d3/

Log:	Fix test for py3.5

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -265,7 +265,8 @@
     def test_interp2app_unwrap_spec_text(self):
         space = self.space
         def g(space, b):
-            return space.newbytes(b)
+            assert isinstance(b, str)
+            return space.newtext(b)
         app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'text'])
         app_g2 = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'text'])
         assert app_g is app_g2

From pypy.commits at gmail.com  Wed Feb 15 06:26:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 03:26:29 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58a43ae5.2ea6df0a.519ef.731b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r862:52ef6e0c9e3d
Date: 2017-02-15 12:26 +0100
http://bitbucket.org/pypy/pypy.org/changeset/52ef6e0c9e3d/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66625 of $105000 (63.5%)
+   $66639 of $105000 (63.5%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2448 left
+    we have $2461 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Feb 15 07:13:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 04:13:55 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix test for py3.5
Message-ID: <58a44603.b6a9df0a.a5c35.812a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90146:1bb0b9c53e51
Date: 2017-02-15 11:37 +0100
http://bitbucket.org/pypy/pypy/changeset/1bb0b9c53e51/

Log:	fix test for py3.5

diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -278,9 +278,9 @@
 def test_isinstance_shortcut():
     from pypy.objspace.std import objspace
     space = objspace.StdObjSpace()
-    w_a = space.wrap("a")
+    w_a = space.newtext("a")
     space.type = None
     # if it crashes, it means that space._type_isinstance didn't go through
     # the fast path, and tries to call type() (which is set to None just
     # above)
-    space.isinstance_w(w_a, space.w_bytes) # does not crash
+    space.isinstance_w(w_a, space.w_text) # does not crash

From pypy.commits at gmail.com  Wed Feb 15 07:13:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 04:13:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: big hg merge 8e2bcfb77e80
Message-ID: <58a44605.2d88df0a.c0b1d.8191@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90147:98a26e7ce3ed
Date: 2017-02-15 13:13 +0100
http://bitbucket.org/pypy/pypy/changeset/98a26e7ce3ed/

Log:	big hg merge 8e2bcfb77e80

diff too long, truncating to 2000 out of 3941 lines

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -13,7 +13,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -1445,7 +1445,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = space.newtext(self.module) if self.module is not None else space.w_None  # identifier
+        w_module = space.newtext_or_none(self.module)  # identifier
         space.setattr(w_node, space.newtext('module'), w_module)
         if self.names is None:
             names_w = []
@@ -1468,7 +1468,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.str_or_None_w(w_module)
+        _module = space.text_or_None_w(w_module)
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -2663,15 +2663,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FormattedValue)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_conversion = space.wrap(self.conversion)  # int
-        space.setattr(w_node, space.wrap('conversion'), w_conversion)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_conversion = space.newint(self.conversion)  # int
+        space.setattr(w_node, space.newtext('conversion'), w_conversion)
         w_format_spec = self.format_spec.to_object(space) if self.format_spec is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('format_spec'), w_format_spec)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('format_spec'), w_format_spec)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2716,11 +2716,11 @@
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2790,9 +2790,9 @@
         w_node = space.call_function(get(space).w_NameConstant)
         w_value = self.value  # singleton
         space.setattr(w_node, space.newtext('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
+        w_lineno = space.newint(self.lineno)  # int
         space.setattr(w_node, space.newtext('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
+        w_col_offset = space.newint(self.col_offset)  # int
         space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
@@ -3755,7 +3755,7 @@
         w_node = space.call_function(get(space).w_ExceptHandler)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
         space.setattr(w_node, space.newtext('type'), w_type)
-        w_name = space.newtext(self.name) if self.name is not None else space.w_None  # identifier
+        w_name = space.newtext_or_none(self.name)  # identifier
         space.setattr(w_node, space.newtext('name'), w_name)
         if self.body is None:
             body_w = []
@@ -3777,7 +3777,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _type = expr.from_object(space, w_type)
-        _name = space.str_or_None_w(w_name)
+        _name = space.text_or_None_w(w_name)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         _lineno = space.int_w(w_lineno)
@@ -3899,9 +3899,9 @@
         space.setattr(w_node, space.newtext('arg'), w_arg)
         w_annotation = self.annotation.to_object(space) if self.annotation is not None else space.w_None  # expr
         space.setattr(w_node, space.newtext('annotation'), w_annotation)
-        w_lineno = space.wrap(self.lineno)  # int
+        w_lineno = space.newint(self.lineno)  # int
         space.setattr(w_node, space.newtext('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
+        w_col_offset = space.newint(self.col_offset)  # int
         space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
@@ -3936,7 +3936,7 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.newtext(self.arg) if self.arg is not None else space.w_None  # identifier
+        w_arg = space.newtext_or_none(self.arg)  # identifier
         space.setattr(w_node, space.newtext('arg'), w_arg)
         w_value = self.value.to_object(space)  # expr
         space.setattr(w_node, space.newtext('value'), w_value)
@@ -3946,7 +3946,7 @@
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', True)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.str_or_None_w(w_arg)
+        _arg = space.text_or_None_w(w_arg)
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
@@ -3970,7 +3970,7 @@
         w_node = space.call_function(get(space).w_alias)
         w_name = space.newtext(self.name)  # identifier
         space.setattr(w_node, space.newtext('name'), w_name)
-        w_asname = space.newtext(self.asname) if self.asname is not None else space.w_None  # identifier
+        w_asname = space.newtext_or_none(self.asname)  # identifier
         space.setattr(w_node, space.newtext('asname'), w_asname)
         return w_node
 
@@ -3981,7 +3981,7 @@
         _name = space.identifier_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.str_or_None_w(w_asname)
+        _asname = space.text_or_None_w(w_asname)
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -127,6 +127,6 @@
         return name
 
     from pypy.module.unicodedata.interp_ucd import ucd
-    w_name = space.wrap(name.decode('utf-8'))
-    w_id = space.call_method(ucd, 'normalize', space.wrap('NFKC'), w_name)
-    return space.unicode_w(w_id).encode('utf-8')
+    w_name = space.newtext(name)
+    w_id = space.call_method(ucd, 'normalize', space.newtext('NFKC'), w_name)
+    return space.text_w(w_id)
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -137,10 +137,9 @@
         elif field.type == "int":
             return "space.newint(%s)" % (value,)
         elif field.type == "identifier":
-            wrapper = "space.newtext(%s)" % (value,)
             if field.opt:
-                wrapper += " if %s is not None else space.w_None" % (value,)
-            return wrapper
+                return "space.newtext_or_none(%s)" % (value,)
+            return "space.newtext(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             allow_none = field.opt
@@ -162,7 +161,7 @@
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
-                return "space.str_or_None_w(%s)" % (value,)
+                return "space.text_or_None_w(%s)" % (value,)
             return "space.identifier_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
@@ -442,7 +441,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -312,8 +312,8 @@
         constructed before there is an object space instance. """
         return self
 
+    @not_rpython
     def unwrap(self, space):
-        """NOT_RPYTHON"""
         # _____ this code is here to support testing only _____
         return self
 
@@ -396,8 +396,9 @@
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
+    @not_rpython
     def __init__(self, config=None):
-        "NOT_RPYTHON: Basic initialization of objects."
+        "Basic initialization of objects."
         self.fromcache = InternalSpaceCache(self).getorbuild
         self.threadlocals = ThreadLocals()
         # set recursion limit
@@ -473,8 +474,9 @@
         except AttributeError:
             return self.__class__.__name__
 
+    @not_rpython
     def setbuiltinmodule(self, importname):
-        """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules"""
+        """load a lazy pypy/module and put it into sys.modules"""
         if '.' in importname:
             fullname = importname
             importname = fullname.rsplit('.', 1)[1]
@@ -532,8 +534,8 @@
             self.setitem(w_modules, w_name, w_mod)
         return w_mod
 
+    @not_rpython
     def get_builtinmodule_to_install(self):
-        """NOT_RPYTHON"""
         try:
             return self._builtinmodule_list
         except AttributeError:
@@ -562,8 +564,9 @@
         'parser', 'fcntl', '_codecs', 'binascii'
     ]
 
+    @not_rpython
     def make_builtins(self):
-        "NOT_RPYTHON: only for initializing the space."
+        "only for initializing the space."
 
         from pypy.module.exceptions import Module
         w_name = self.newtext('__exceptions__')
@@ -621,8 +624,8 @@
         objects."""
         raise NotImplementedError
 
+    @not_rpython
     def export_builtin_exceptions(self):
-        """NOT_RPYTHON"""
         w_dic = self.exceptions_module.getdict(self)
         exc_types_w = {}
         w_iter = self.iter(w_dic)
@@ -641,8 +644,8 @@
                 setattr(self, "w_" + excname, w_exc)
         return exc_types_w
 
+    @not_rpython
     def install_mixedmodule(self, mixedname, installed_builtin_modules):
-        """NOT_RPYTHON"""
         modname = self.setbuiltinmodule(mixedname)
         if modname:
             assert modname not in installed_builtin_modules, (
@@ -650,8 +653,9 @@
                 "app-level module %r" % (modname,))
             installed_builtin_modules.append(modname)
 
+    @not_rpython
     def setup_builtin_modules(self):
-        "NOT_RPYTHON: only for initializing the space."
+        "only for initializing the space."
         if self.config.objspace.usemodules.cpyext:
             # Special-case this to have state.install_dll() called early, which
             # is required to initialise sys on Windows.
@@ -664,8 +668,9 @@
         for mod in self.builtin_modules.values():
             mod.setup_after_space_initialization()
 
+    @not_rpython
     def initialize(self):
-        """NOT_RPYTHON: Abstract method that should put some minimal
+        """Abstract method that should put some minimal
         content into the w_builtins."""
 
     def getexecutioncontext(self):
@@ -1260,14 +1265,16 @@
     def exception_issubclass_w(self, w_cls1, w_cls2):
         return self.issubtype_w(w_cls1, w_cls2)
 
+    @not_rpython
     def new_exception_class(self, *args, **kwargs):
-        "NOT_RPYTHON; convenience method to create excceptions in modules"
+        "convenience method to create excceptions in modules"
         return new_exception_class(self, *args, **kwargs)
 
     # end of special support code
 
+    @not_rpython
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
-        "NOT_RPYTHON: For internal debugging."
+        "For internal debugging."
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1276,9 +1283,10 @@
             raise TypeError('space.eval(): expected a string, code or PyCode object')
         return expression.exec_code(self, w_globals, w_locals)
 
+    @not_rpython
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
-        "NOT_RPYTHON: For internal debugging."
+        "For internal debugging."
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1509,7 +1517,7 @@
                 return None
             code = 's*'
         if code == 's*':
-            if self.isinstance_w(w_obj, self.w_str):
+            if self.isinstance_w(w_obj, self.w_bytes):
                 return StringBuffer(w_obj.bytes_w(self))
             if self.isinstance_w(w_obj, self.w_unicode):
                 return StringBuffer(w_obj.identifier_w(self))  # no surrogates
@@ -1518,7 +1526,7 @@
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or buffer", w_obj)
         elif code == 's#':
-            if self.isinstance_w(w_obj, self.w_str):
+            if self.isinstance_w(w_obj, self.w_bytes):
                 return w_obj.bytes_w(self)
             if self.isinstance_w(w_obj, self.w_unicode):
                 return w_obj.identifier_w(self)    # no surrogates (forbidden)
@@ -1559,8 +1567,6 @@
                 raise
         return self.buffer_w(w_obj, flags).as_str()
 
-    def str_or_None_w(self, w_obj):
-        return None if self.is_none(w_obj) else self.str_w(w_obj)
 
     def text_or_None_w(self, w_obj):
         return None if self.is_none(w_obj) else self.text_w(w_obj)
@@ -1852,8 +1858,8 @@
 
 
 class AppExecCache(SpaceCache):
+    @not_rpython
     def build(cache, source):
-        """ NOT_RPYTHON """
         space = cache.space
         # XXX will change once we have our own compiler
         import py
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -27,6 +27,9 @@
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
 
+from rpython.rlib.signature import signature, finishsigs
+from rpython.rlib import types as sigtypes
+
 NO_DEFAULT = object()
 
 
@@ -153,7 +156,7 @@
     def visit_bufferstr(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
-    def visit_str_or_None(self, el, app_sig):
+    def visit_str_or_None(self, el, app_sig):   # XXX kill me
         self.checked_space_method(el, app_sig)
 
     def visit_text_or_None(self, el, app_sig):
@@ -165,6 +168,9 @@
     def visit_bytes(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_text(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_fsencode(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -296,8 +302,8 @@
     def visit_bufferstr(self, typ):
         self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),))
 
-    def visit_str_or_None(self, typ):
-        self.run_args.append("space.str_or_None_w(%s)" % (self.scopenext(),))
+    def visit_str_or_None(self, typ):  #XXX kill me
+        self.run_args.append("space.text_or_None_w(%s)" % (self.scopenext(),))
 
     def visit_text_or_None(self, typ):
         self.run_args.append("space.text_or_None_w(%s)" % (self.scopenext(),))
@@ -308,6 +314,9 @@
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
 
+    def visit_text(self, typ):
+        self.run_args.append("space.text_w(%s)" % (self.scopenext(),))
+
     def visit_fsencode(self, typ):
         self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
 
@@ -458,8 +467,8 @@
     def visit_bufferstr(self, typ):
         self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),))
 
-    def visit_str_or_None(self, typ):
-        self.unwrap.append("space.str_or_None_w(%s)" % (self.nextarg(),))
+    def visit_str_or_None(self, typ):  #XXX kill me
+        self.unwrap.append("space.text_or_None_w(%s)" % (self.nextarg(),))
 
     def visit_text_or_None(self, typ):
         self.unwrap.append("space.text_or_None_w(%s)" % (self.nextarg(),))
@@ -470,6 +479,9 @@
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
 
+    def visit_text(self, typ):
+        self.unwrap.append("space.text_w(%s)" % (self.nextarg(),))
+
     def visit_fsencode(self, typ):
         self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
 
@@ -839,11 +851,17 @@
             w_result = space.w_None
         return w_result
 
+w_root_or_none = sigtypes.instance(W_Root, can_be_None=True)
 
+ at finishsigs
 class BuiltinCode1(BuiltinCode):
     _immutable_ = True
     fast_natural_arity = 1
 
+    @signature(sigtypes.self(), sigtypes.any(),
+               w_root_or_none,
+               w_root_or_none,
+               returns=w_root_or_none)
     def fastcall_1(self, space, w_func, w1):
         try:
             w_result = self.fastfunc_1(space, w1)
@@ -860,10 +878,16 @@
         return w_result
 
 
+ at finishsigs
 class BuiltinCode2(BuiltinCode):
     _immutable_ = True
     fast_natural_arity = 2
 
+    @signature(sigtypes.self(), sigtypes.any(),
+               w_root_or_none,
+               w_root_or_none,
+               w_root_or_none,
+               returns=w_root_or_none)
     def fastcall_2(self, space, w_func, w1, w2):
         try:
             w_result = self.fastfunc_2(space, w1, w2)
@@ -880,10 +904,17 @@
         return w_result
 
 
+ at finishsigs
 class BuiltinCode3(BuiltinCode):
     _immutable_ = True
     fast_natural_arity = 3
 
+    @signature(sigtypes.self(), sigtypes.any(),
+               w_root_or_none,
+               w_root_or_none,
+               w_root_or_none,
+               w_root_or_none,
+               returns=w_root_or_none)
     def fastcall_3(self, space, func, w1, w2, w3):
         try:
             w_result = self.fastfunc_3(space, w1, w2, w3)
@@ -899,12 +930,20 @@
             w_result = space.w_None
         return w_result
 
-
+ at finishsigs
 class BuiltinCode4(BuiltinCode):
     _immutable_ = True
     fast_natural_arity = 4
 
+    @signature(sigtypes.self(), sigtypes.any(),
+               w_root_or_none,
+               w_root_or_none,
+               w_root_or_none,
+               w_root_or_none,
+               w_root_or_none,
+               returns=w_root_or_none)
     def fastcall_4(self, space, func, w1, w2, w3, w4):
+        from rpython.rlib.debug import check_annotation
         try:
             w_result = self.fastfunc_4(space, w1, w2, w3, w4)
         except DescrMismatch:
@@ -1025,7 +1064,9 @@
             if defaultval is not NO_DEFAULT:
                 if name != '__args__' and name != 'args_w':
                     if w_def is Ellipsis:
-                        if isinstance(defaultval, str) and spec not in [str]:
+                        if isinstance(defaultval, str) and (
+                                # XXX hackish
+                           spec == 'bytes' or isinstance(spec, WrappedDefault)):
                             w_def = space.newbytes(defaultval)
                         else:
                             w_def = space.wrap(defaultval)
diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -152,7 +152,7 @@
                                if not k.startswith('w_')]))
             del local['locals']
             for w_name in self.space.unpackiterable(self.w_globals):
-                local['w_' + self.space.str_w(w_name)] = (
+                local['w_' + self.space.text_w(w_name)] = (
                     self.space.getitem(self.w_globals, w_name))
             code.interact(banner=banner, local=local)
             # copy back 'w_' names
@@ -169,13 +169,12 @@
 
     def runsource(self, source, ignored_filename="<input>", symbol="single"):
         # the following hacked file name is recognized specially by error.py
-        hacked_filename = '<inline>\n' + source.encode(
-                                            'ascii', 'backslashreplace')
         compiler = self.space.getexecutioncontext().compiler
 
         # CPython 2.6 turns console input into unicode
         if isinstance(source, unicode):
             source = source.encode(sys.stdin.encoding)
+        hacked_filename = '<inline>\n' + source
 
         def doit():
             # compile the provided input
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -125,7 +125,7 @@
                         exitcode = space.int_w(w_exitcode, allow_conversion=False)
                     except OperationError:
                         # not an integer: print it to stderr
-                        msg = space.str_w(space.str(w_exitcode))
+                        msg = space.text_w(space.str(w_exitcode))
                         print >> sys.stderr, msg
                         exitcode = 1
                 raise SystemExit(exitcode)
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -88,7 +88,7 @@
     def descr__reduce__(self, space):
         w_name = space.finditem(self.w_dict, space.newtext('__name__'))
         if (w_name is None or
-            not space.isinstance_w(w_name, space.w_unicode)):
+            not space.isinstance_w(w_name, space.w_text)):
             # maybe raise exception here (XXX this path is untested)
             return space.w_None
         w_modules = space.sys.get('modules')
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -25,7 +25,7 @@
 
 # helper
 
-def unpack_str_tuple(space,w_str_tuple):
+def unpack_text_tuple(space,w_str_tuple):
     return [space.text_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
 
@@ -375,9 +375,9 @@
         return w_result
 
     @unwrap_spec(argcount=int, kwonlyargcount=int, nlocals=int, stacksize=int, flags=int,
-                 codestring=str,
-                 filename='str0', name=str, firstlineno=int,
-                 lnotab=str, magic=int)
+                 codestring='bytes',
+                 filename='str0', name='text', firstlineno=int,
+                 lnotab='bytes', magic=int)
     def descr_code__new__(space, w_subtype,
                           argcount, kwonlyargcount, nlocals, stacksize, flags,
                           codestring, w_constants, w_names,
@@ -396,14 +396,14 @@
         if not space.isinstance_w(w_constants, space.w_tuple):
             raise oefmt(space.w_TypeError, "Expected tuple for constants")
         consts_w = space.fixedview(w_constants)
-        names = unpack_str_tuple(space, w_names)
-        varnames = unpack_str_tuple(space, w_varnames)
+        names = unpack_text_tuple(space, w_names)
+        varnames = unpack_text_tuple(space, w_varnames)
         if w_freevars is not None:
-            freevars = unpack_str_tuple(space, w_freevars)
+            freevars = unpack_text_tuple(space, w_freevars)
         else:
             freevars = []
         if w_cellvars is not None:
-            cellvars = unpack_str_tuple(space, w_cellvars)
+            cellvars = unpack_text_tuple(space, w_cellvars)
         else:
             cellvars = []
         code = space.allocate_instance(PyCode, w_subtype)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -857,7 +857,7 @@
 
 def unpickle_block(space, w_tup):
     w_opname, w_handlerposition, w_valuestackdepth = space.unpackiterable(w_tup)
-    opname = space.str_w(w_opname)
+    opname = space.text_w(w_opname)
     handlerposition = space.int_w(w_handlerposition)
     valuestackdepth = space.int_w(w_valuestackdepth)
     assert valuestackdepth >= 0
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -144,7 +144,7 @@
                 if e.match(space, space.w_UnicodeDecodeError):
                     e.normalize_exception(space)
                     w_message = space.str(e.get_w_value(space))
-                    raise error.SyntaxError(space.str_w(w_message))
+                    raise error.SyntaxError(space.text_w(w_message))
                 raise
 
         flags = compile_info.flags
diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -82,7 +82,7 @@
         w_ret = parsestring.parsestr(space, None, repr("hello"))
         assert space.isinstance_w(w_ret, space.w_unicode)
         w_ret = parsestring.parsestr(space, None, "b'hi'")
-        assert space.isinstance_w(w_ret, space.w_str)
+        assert space.isinstance_w(w_ret, space.w_bytes)
         w_ret = parsestring.parsestr(space, None, "r'hi'")
         assert space.isinstance_w(w_ret, space.w_unicode)
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -260,7 +260,6 @@
         # we can't use the "bytes" object for the unwrap_spec, because that's
         # an alias for "str" on the underlying Python2
         space = self.space
-        w = space.wrap
         def g(space, b):
             return space.newbytes(b)
         app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'bytes'])
@@ -270,6 +269,18 @@
         assert self.space.eq_w(space.call_function(w_app_g, space.newbytes("abc")),
                                space.newbytes("abc"))
 
+    def test_interp2app_unwrap_spec_text(self):
+        space = self.space
+        def g(space, b):
+            assert isinstance(b, str)
+            return space.newtext(b)
+        app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'text'])
+        app_g2 = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'text'])
+        assert app_g is app_g2
+        w_app_g = space.wrap(app_g)
+        assert self.space.eq_w(space.call_function(w_app_g, space.newtext("abc")),
+                               space.newtext("abc"))
+
     def test_caching_methods(self):
         class Base(gateway.W_Root):
             def f(self):
@@ -583,7 +594,7 @@
         space = self.space
         w = space.wrap
         def g_run(space, w_type):
-            assert space.is_w(w_type, space.w_str)
+            assert space.is_w(w_type, space.w_text)
             return w(42)
 
         app_g_run = gateway.interp2app_temp(g_run,
@@ -591,7 +602,7 @@
                                                          gateway.W_Root],
                                             as_classmethod=True)
         w_app_g_run = space.wrap(app_g_run)
-        w_bound = space.get(w_app_g_run, w("hello"), space.w_str)
+        w_bound = space.get(w_app_g_run, w("hello"), space.w_text)
         assert space.eq_w(space.call_function(w_bound), w(42))
 
     def test_interp2app_fastcall(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -21,9 +21,9 @@
         w_result = space.isinstance(w_i, space.w_int)
         assert space.is_true(w_result)
         assert space.isinstance_w(w_i, space.w_int)
-        w_result = space.isinstance(w_i, space.w_str)
+        w_result = space.isinstance(w_i, space.w_bytes)
         assert not space.is_true(w_result)
-        assert not space.isinstance_w(w_i, space.w_str)
+        assert not space.isinstance_w(w_i, space.w_bytes)
 
     def test_newlist(self):
         w = self.space.wrap
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -10,7 +10,7 @@
 from pypy.interpreter.nestedscope import Cell
 from pypy.interpreter.function import Function
 
- at unwrap_spec(filename='fsencode', mode=str, flags=int, dont_inherit=int,
+ at unwrap_spec(filename='fsencode', mode='text', flags=int, dont_inherit=int,
              optimize=int)
 def compile(space, w_source, filename, mode, flags=0, dont_inherit=0,
             optimize=0):
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -44,8 +44,8 @@
     # Note that if w_name is already an exact string it must be returned
     # unmodified (and not e.g. unwrapped-rewrapped).
     if not space.is_w(space.type(w_name), space.w_text):
-        name = space.text_w(w_name)  # typecheck
-        w_name = space.newtext(name)  # rewrap as a real string
+        name = space.text_w(w_name)    # typecheck
+        w_name = space.newtext(name)     # rewrap as a real string
     return w_name
 
 def delattr(space, w_object, w_name):
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -7,6 +7,10 @@
 
 
 def create_builder(name, strtype, builder_cls):
+    if strtype is str:
+        unwrap = 'bytes'
+    else:
+        unwrap = unicode
     class W_Builder(W_Root):
         def __init__(self, space, size):
             if size < 0:
@@ -23,12 +27,12 @@
         def descr__new__(space, w_subtype, size=-1):
             return W_Builder(space, size)
 
-        @unwrap_spec(s=strtype)
+        @unwrap_spec(s=unwrap)
         def descr_append(self, space, s):
             self._check_done(space)
             self.builder.append(s)
 
-        @unwrap_spec(s=strtype, start=int, end=int)
+        @unwrap_spec(s=unwrap, start=int, end=int)
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py
--- a/pypy/module/__pypy__/interp_debug.py
+++ b/pypy/module/__pypy__/interp_debug.py
@@ -3,22 +3,22 @@
 
 
 @jit.dont_look_inside
- at unwrap_spec(category=str)
+ at unwrap_spec(category='text')
 def debug_start(space, category):
     debug.debug_start(category)
 
 @jit.dont_look_inside
 def debug_print(space, args_w):
-    parts = [space.str_w(space.str(w_item)) for w_item in args_w]
+    parts = [space.text_w(space.str(w_item)) for w_item in args_w]
     debug.debug_print(' '.join(parts))
 
 @jit.dont_look_inside
- at unwrap_spec(category=str)
+ at unwrap_spec(category='text')
 def debug_stop(space, category):
     debug.debug_stop(category)
 
 
- at unwrap_spec(category=str)
+ at unwrap_spec(category='text')
 def debug_print_once(space, category, args_w):
     debug_start(space, category)
     debug_print(space, args_w)
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec
 
- at unwrap_spec(type=str)
+ at unwrap_spec(type='text')
 def newdict(space, type):
     """ newdict(type)
 
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -22,7 +22,7 @@
     attach_gdb()
 
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def method_cache_counter(space, name):
     """Return a tuple (method_cache_hits, method_cache_misses) for calls to
     methods with the name."""
@@ -41,7 +41,7 @@
     cache.misses = {}
     cache.hits = {}
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def mapdict_cache_counter(space, name):
     """Return a tuple (index_cache_hits, index_cache_misses) for lookups
     in the mapdict cache with the given attribute name."""
@@ -68,7 +68,7 @@
     func.getcode().hidden_applevel = True
     return w_func
 
- at unwrap_spec(meth=str)
+ at unwrap_spec(meth='text')
 def lookup_special(space, w_obj, meth):
     """Lookup up a special method on an object."""
     w_descr = space.lookup(w_obj, meth)
@@ -143,7 +143,7 @@
     else:
         cache._code_hook = w_callable
 
- at unwrap_spec(string=str, byteorder=str, signed=int)
+ at unwrap_spec(string='bytes', byteorder='text', signed=int)
 def decode_long(space, string, byteorder='little', signed=1):
     from rpython.rlib.rbigint import rbigint, InvalidEndiannessError
     try:
@@ -163,8 +163,8 @@
         jit.promote(space.int_w(w_obj))
     elif space.is_w(space.type(w_obj), space.w_float):
         jit.promote(space.float_w(w_obj))
-    elif space.is_w(space.type(w_obj), space.w_str):
-        jit.promote_string(space.str_w(w_obj))
+    elif space.is_w(space.type(w_obj), space.w_bytes):
+        jit.promote_string(space.bytes_w(w_obj))
     elif space.is_w(space.type(w_obj), space.w_unicode):
         raise oefmt(space.w_TypeError, "promoting unicode unsupported")
     else:
diff --git a/pypy/module/_cffi_backend/call_python.py b/pypy/module/_cffi_backend/call_python.py
--- a/pypy/module/_cffi_backend/call_python.py
+++ b/pypy/module/_cffi_backend/call_python.py
@@ -91,7 +91,7 @@
 
     if space.is_w(w_name, space.w_None):
         w_name = space.getattr(w_python_callable, space.newtext('__name__'))
-    name = space.str_w(w_name)
+    name = space.text_w(w_name)
 
     ctx = ffi.ctxobj.ctx
     index = parse_c_type.search_in_globals(ctx, name)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -98,7 +98,7 @@
 
     def _repr_extra(self):
         space = self.space
-        return 'calling ' + space.str_w(space.repr(self.w_callable))
+        return 'calling ' + space.text_w(space.repr(self.w_callable))
 
     def write_error_return_value(self, ll_res):
         error_string = self.error_string
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -232,10 +232,10 @@
         from pypy.module._cffi_backend import ctypeprim
         space = self.space
         if isinstance(ctitem, ctypeprim.W_CTypePrimitive) and ctitem.size == 1:
-            if space.isinstance_w(w_value, space.w_str):
+            if space.isinstance_w(w_value, space.w_bytes):
                 from rpython.rtyper.annlowlevel import llstr
                 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-                value = space.str_w(w_value)
+                value = space.bytes_w(w_value)
                 if len(value) != length:
                     raise oefmt(space.w_ValueError,
                                 "need a string of length %d, got %d",
@@ -325,7 +325,7 @@
 
     def getcfield(self, w_attr, mode):
         space = self.space
-        attr = space.str_w(w_attr)
+        attr = space.text_w(w_attr)
         try:
             cfield = self.ctype.getcfield(attr)
         except KeyError:
@@ -430,8 +430,8 @@
         with self as ptr:
             if not ptr:
                 raise oefmt(space.w_RuntimeError,
-                            "cannot use unpack() on %s",
-                            space.str_w(self.repr()))
+                            "cannot use unpack() on %R",
+                            self)
             w_result = ctype.ctitem.unpack_ptr(ctype, ptr, length)
         return w_result
 
@@ -576,7 +576,7 @@
 
     def _repr_extra(self):
         w_repr = self.space.repr(self.w_keepalive)
-        return "handle to %s" % (self.space.str_w(w_repr),)
+        return "handle to %s" % (self.space.text_w(w_repr),)
 
 
 class W_CDataFromBuffer(W_CData):
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -155,7 +155,7 @@
         p = rffi.ptradd(p, llmemory.raw_malloc_usage(n * rffi.sizeof(GLOBAL_S)))
         nintconsts = rffi.cast(rffi.CArrayPtr(CDL_INTCONST_S), p)
         for i in range(n):
-            decoder = StringDecoder(ffi, space.str_w(globals_w[i * 2]))
+            decoder = StringDecoder(ffi, space.bytes_w(globals_w[i * 2]))
             nglobs[i].c_type_op = decoder.next_opcode()
             nglobs[i].c_name = decoder.next_name()
             op = getop(nglobs[i].c_type_op)
@@ -185,7 +185,7 @@
             # 'desc' is the tuple of strings (desc_struct, desc_field_1, ..)
             desc = space.fixedview(struct_unions_w[i])
             nf1 = len(desc) - 1
-            decoder = StringDecoder(ffi, space.str_w(desc[0]))
+            decoder = StringDecoder(ffi, space.bytes_w(desc[0]))
             rffi.setintfield(nstructs[i], 'c_type_index', decoder.next_4bytes())
             flags = decoder.next_4bytes()
             rffi.setintfield(nstructs[i], 'c_flags', flags)
@@ -202,7 +202,7 @@
                 rffi.setintfield(nstructs[i], 'c_first_field_index', nf)
                 rffi.setintfield(nstructs[i], 'c_num_fields', nf1)
             for j in range(nf1):
-                decoder = StringDecoder(ffi, space.str_w(desc[j + 1]))
+                decoder = StringDecoder(ffi, space.bytes_w(desc[j + 1]))
                 # this 'decoder' is for one of the other strings beyond
                 # the first one, describing one field each
                 type_op = decoder.next_opcode()
@@ -226,7 +226,7 @@
         n = len(enums_w)
         nenums = allocate_array(ffi, ENUM_S, n)
         for i in range(n):
-            decoder = StringDecoder(ffi, space.str_w(enums_w[i]))
+            decoder = StringDecoder(ffi, space.bytes_w(enums_w[i]))
             rffi.setintfield(nenums[i], 'c_type_index', decoder.next_4bytes())
             rffi.setintfield(nenums[i], 'c_type_prim', decoder.next_4bytes())
             nenums[i].c_name = decoder.next_name()
@@ -241,7 +241,7 @@
         n = len(typenames_w)
         ntypenames = allocate_array(ffi, TYPENAME_S, n)
         for i in range(n):
-            decoder = StringDecoder(ffi, space.str_w(typenames_w[i]))
+            decoder = StringDecoder(ffi, space.bytes_w(typenames_w[i]))
             rffi.setintfield(ntypenames[i],'c_type_index',decoder.next_4bytes())
             ntypenames[i].c_name = decoder.next_name()
         ffi.ctxobj.ctx.c_typenames = ntypenames
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -175,7 +175,7 @@
     def direct_typeoffsetof(self, w_field_or_index, following=0):
         space = self.space
         try:
-            fieldname = space.str_w(w_field_or_index)
+            fieldname = space.text_w(w_field_or_index)
         except OperationError as e:
             if not e.match(space, space.w_TypeError):
                 raise
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -25,14 +25,14 @@
 
     def extra_repr(self, cdata):
         w_ob = self.convert_to_object(cdata)
-        return self.space.str_w(self.space.repr(w_ob))
+        return self.space.text_w(self.space.repr(w_ob))
 
     def _alignof(self):
         return self.align
 
     def cast_str(self, w_ob):
         space = self.space
-        s = space.str_w(w_ob)
+        s = space.bytes_w(w_ob)
         if len(s) != 1:
             raise oefmt(space.w_TypeError,
                         "cannot cast string of length %d to ctype '%s'",
@@ -56,7 +56,7 @@
             ptr = w_ob.unsafe_escaping_ptr()
             value = rffi.cast(lltype.Signed, ptr)
             value = self._cast_result(value)
-        elif space.isinstance_w(w_ob, space.w_str):
+        elif space.isinstance_w(w_ob, space.w_bytes):
             value = self.cast_str(w_ob)
             value = self._cast_result(value)
         elif space.isinstance_w(w_ob, space.w_unicode):
@@ -76,7 +76,7 @@
 
     def _overflow(self, w_ob):
         space = self.space
-        s = space.str_w(space.str(w_ob))
+        s = space.text_w(space.str(w_ob))
         raise oefmt(space.w_OverflowError,
                     "integer %s does not fit '%s'", s, self.name)
 
@@ -396,7 +396,7 @@
                             w_ob.ctype.name, self.name)
             w_ob = w_ob.convert_to_object()
         #
-        if space.isinstance_w(w_ob, space.w_str):
+        if space.isinstance_w(w_ob, space.w_bytes):
             value = self.cast_str(w_ob)
         elif space.isinstance_w(w_ob, space.w_unicode):
             value = self.cast_unicode(w_ob)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -76,9 +76,9 @@
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
         elif self.accept_str:
-            if not space.isinstance_w(w_ob, space.w_str):
+            if not space.isinstance_w(w_ob, space.w_bytes):
                 raise self._convert_error("bytes or list or tuple", w_ob)
-            s = space.str_w(w_ob)
+            s = space.bytes_w(w_ob)
             n = len(s)
             if self.length >= 0 and n > self.length:
                 raise oefmt(space.w_IndexError,
@@ -118,8 +118,8 @@
             with cdataobj as ptr:
                 if not ptr:
                     raise oefmt(space.w_RuntimeError,
-                                "cannot use string() on %s",
-                                space.str_w(cdataobj.repr()))
+                                "cannot use string() on %R",
+                                cdataobj)
                 #
                 from pypy.module._cffi_backend import ctypearray
                 length = maxlen
@@ -291,7 +291,7 @@
 
     def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
         space = self.space
-        if self.accept_str and space.isinstance_w(w_init, space.w_str):
+        if self.accept_str and space.isinstance_w(w_init, space.w_bytes):
             # special case to optimize strings passed to a "char *" argument
             value = space.bytes_w(w_init)
             if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool):
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -139,7 +139,7 @@
             lst_w = space.fixedview(w_ob)
             for i in range(len(lst_w)):
                 w_key = lst_w[i]
-                key = space.str_w(w_key)
+                key = space.text_w(w_key)
                 try:
                     cf = self._fields_dict[key]
                 except KeyError:
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -151,8 +151,8 @@
     def ffi_type(self, w_x, accept):
         space = self.space
         if (accept & ACCEPT_STRING) and (
-                space.isinstance_w(w_x, space.w_unicode)):
-            string = space.str_w(w_x)
+                space.isinstance_w(w_x, space.w_text)):
+            string = space.text_w(w_x)
             consider_fn_as_fnptr = (accept & CONSIDER_FN_AS_FNPTR) != 0
             if jit.isconstant(string):
                 try:
@@ -174,7 +174,7 @@
                     m1, s12, m2, s23, m3, w_x)
 
 
-    @unwrap_spec(module_name=str, _version=int, _types=str)
+    @unwrap_spec(module_name='text', _version=int, _types='text')
     def descr_init(self, module_name='?', _version=-1, _types='',
                    w__globals=None, w__struct_unions=None, w__enums=None,
                    w__typenames=None, w__includes=None):
@@ -226,7 +226,7 @@
         space = self.space
         if isinstance(w_arg, W_LibObject) and len(args_w) == 1:
             # case 3 in the docstring
-            return w_arg.address_of_func_or_global_var(space.str_w(args_w[0]))
+            return w_arg.address_of_func_or_global_var(space.text_w(args_w[0]))
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_CDATA)
         if len(args_w) == 0:
@@ -361,7 +361,7 @@
         return w_cdata.with_gc(w_destructor)
 
 
-    @unwrap_spec(replace_with=str)
+    @unwrap_spec(replace_with='text')
     def descr_getctype(self, w_cdecl, replace_with=''):
         """\
 Return a string giving the C type 'cdecl', which may be itself a
@@ -598,7 +598,7 @@
         lib.cdlopen_close()
 
 
-    @unwrap_spec(name=str)
+    @unwrap_spec(name='text')
     def descr_integer_const(self, name):
         """\
 Get the value of an integer constant.
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -64,7 +64,7 @@
 
 # ____________________________________________________________
 
- at unwrap_spec(w_ctype=ctypeobj.W_CType, replace_with=str)
+ at unwrap_spec(w_ctype=ctypeobj.W_CType, replace_with='text')
 def getcname(space, w_ctype, replace_with):
     p = w_ctype.name_position
     s = '%s%s%s' % (w_ctype.name[:p], replace_with, w_ctype.name[p:])
@@ -140,7 +140,7 @@
         raise oefmt(space.w_TypeError,
                         "from_buffer() cannot return the address a unicode")
     buf = _fetch_as_read_buffer(space, w_x)
-    if space.isinstance_w(w_x, space.w_str):
+    if space.isinstance_w(w_x, space.w_bytes):
         _cdata = get_raw_address_of_string(space, w_x)
     else:
         try:
@@ -181,7 +181,7 @@
     cache = space.fromcache(RawBytesCache)
     rawbytes = cache.wdict.get(w_x)
     if rawbytes is None:
-        data = space.str_w(w_x)
+        data = space.bytes_w(w_x)
         if we_are_translated() and not rgc.can_move(data):
             lldata = llstr(data)
             data_start = (llmemory.cast_ptr_to_adr(lldata) +
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -222,7 +222,7 @@
         else:
             raise oefmt(self.space.w_AttributeError,
                         "cannot write to function or constant '%s'",
-                        self.space.str_w(w_attr))
+                        self.space.text_w(w_attr))
 
     def descr_delattr(self, w_attr):
         self._get_attr(w_attr)    # for the possible AttributeError
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -38,7 +38,7 @@
         space = self.space
         return space.newtext("<clibrary '%s'>" % self.name)
 
-    @unwrap_spec(w_ctype=W_CType, name=str)
+    @unwrap_spec(w_ctype=W_CType, name='text')
     def load_function(self, w_ctype, name):
         from pypy.module._cffi_backend import ctypeptr, ctypearray
         space = self.space
@@ -58,7 +58,7 @@
             w_ctype = w_ctype.ctptr
         return W_CData(space, rffi.cast(rffi.CCHARP, cdata), w_ctype)
 
-    @unwrap_spec(w_ctype=W_CType, name=str)
+    @unwrap_spec(w_ctype=W_CType, name='text')
     def read_variable(self, w_ctype, name):
         space = self.space
         try:
@@ -69,7 +69,7 @@
                         name, self.name)
         return w_ctype.convert_to_object(rffi.cast(rffi.CCHARP, cdata))
 
-    @unwrap_spec(w_ctype=W_CType, name=str)
+    @unwrap_spec(w_ctype=W_CType, name='text')
     def write_variable(self, w_ctype, name, w_value):
         space = self.space
         try:
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -153,7 +153,7 @@
     eptypesize("int_fast64_t",  8, _WCTSigned)
     eptypesize("uint_fast64_t", 8, _WCTUnsign)
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def new_primitive_type(space, name):
     return _new_primitive_type(space, name)
 
@@ -280,11 +280,11 @@
 # ____________________________________________________________
 
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def new_struct_type(space, name):
     return ctypestruct.W_CTypeStruct(space, name)
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def new_union_type(space, name):
     return ctypestruct.W_CTypeUnion(space, name)
 
@@ -331,7 +331,7 @@
         field_w = space.fixedview(w_field)
         if not (2 <= len(field_w) <= 4):
             raise oefmt(space.w_TypeError, "bad field descr")
-        fname = space.str_w(field_w[0])
+        fname = space.text_w(field_w[0])
         ftype = space.interp_w(ctypeobj.W_CType, field_w[1])
         fbitsize = -1
         foffset = -1
@@ -590,13 +590,13 @@
 
 # ____________________________________________________________
 
- at unwrap_spec(name=str, w_basectype=ctypeobj.W_CType)
+ at unwrap_spec(name='text', w_basectype=ctypeobj.W_CType)
 def new_enum_type(space, name, w_enumerators, w_enumvalues, w_basectype):
     enumerators_w = space.fixedview(w_enumerators)
     enumvalues_w  = space.fixedview(w_enumvalues)
     if len(enumerators_w) != len(enumvalues_w):
         raise oefmt(space.w_ValueError, "tuple args must have the same size")
-    enumerators = [space.str_w(w) for w in enumerators_w]
+    enumerators = [space.text_w(w) for w in enumerators_w]
     #
     if (not isinstance(w_basectype, ctypeprim.W_CTypePrimitiveSigned) and
         not isinstance(w_basectype, ctypeprim.W_CTypePrimitiveUnsigned)):
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -145,7 +145,7 @@
         raise oefmt(space.w_TypeError, "argument must be callable")
 
 
- at unwrap_spec(encoding=str)
+ at unwrap_spec(encoding='text')
 def lookup_codec(space, encoding):
     """lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer)
     Looks up a codec tuple in the Python codec registry and returns
@@ -207,7 +207,7 @@
         raise oefmt(space.w_TypeError, "wrong exception")
 
     delta = space.int_w(w_end) - space.int_w(w_start)
-    if delta < 0 or not (space.isinstance_w(w_obj, space.w_str) or
+    if delta < 0 or not (space.isinstance_w(w_obj, space.w_bytes) or
                          space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError, "wrong exception")
 
@@ -504,7 +504,7 @@
                     "encoder must return a tuple (object, integer)")
     return space.getitem(w_res, space.newint(0))
 
- at unwrap_spec(errors=str)
+ at unwrap_spec(errors='text')
 def lookup_error(space, errors):
     """lookup_error(errors) -> handler
 
@@ -521,7 +521,7 @@
     return w_err_handler
 
 
- at unwrap_spec(errors=str)
+ at unwrap_spec(errors='text')
 def encode(space, w_obj, w_encoding=None, errors='strict'):
     """encode(obj, [encoding[,errors]]) -> object
 
@@ -544,7 +544,7 @@
     s = space.getarg_w('s#', w_data)
     return space.newtuple([space.newbytes(s), space.newint(len(s))])
 
- at unwrap_spec(errors=str)
+ at unwrap_spec(errors='text')
 def decode(space, w_obj, w_encoding=None, errors='strict'):
     """decode(obj, [encoding[,errors]]) -> object
 
@@ -562,7 +562,7 @@
     w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1))
     return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
 
- at unwrap_spec(errors=str)
+ at unwrap_spec(errors='text')
 def register_error(space, errors, w_handler):
     """register_error(errors, handler)
 
@@ -984,9 +984,8 @@
 # support for the "string escape" translation
 # This is a bytes-to bytes transformation
 
- at unwrap_spec(errors='str_or_None')
-def escape_encode(space, w_data, errors='strict'):
-    data = space.bytes_w(w_data)
+ at unwrap_spec(data='bytes', errors='str_or_None')
+def escape_encode(space, data, errors='strict'):
     from pypy.objspace.std.bytesobject import string_escape_encode
     result = string_escape_encode(data, False)
     return space.newtuple([space.newbytes(result), space.newint(len(data))])
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -117,7 +117,7 @@
         length = rwbuffer.getlength()
         w_data = space.call_method(self, methodname, space.newint(length))
 
-        if not space.isinstance_w(w_data, space.w_str):
+        if not space.isinstance_w(w_data, space.w_bytes):
             raise oefmt(space.w_TypeError, "%s() should return bytes",
                         methodname)
         data = space.bytes_w(w_data)
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -143,7 +143,7 @@
         W_FileIO.__init__(self, space)
         return self
 
-    @unwrap_spec(mode=str, closefd=int)
+    @unwrap_spec(mode='text', closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
         if self.fd >= 0:
             if self.closefd:
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -14,7 +14,7 @@
             "io.UnsupportedOperation",
             space.newtuple([space.w_ValueError, space.w_IOError]))
 
- at unwrap_spec(mode=str, buffering=int,
+ at unwrap_spec(mode='text', buffering=int,
              encoding="str_or_None", errors="str_or_None",
              newline="str_or_None", closefd=int)
 def open(space, w_file, mode="r", buffering=-1, encoding=None, errors=None,
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -218,7 +218,7 @@
                     if trap_eintr(space, e):
                         continue
                     raise
-                if not space.isinstance_w(w_readahead, space.w_str):
+                if not space.isinstance_w(w_readahead, space.w_bytes):
                     raise oefmt(space.w_IOError,
                                 "peek() should have returned a bytes object, "
                                 "not '%T'", w_readahead)
@@ -248,7 +248,7 @@
                 if trap_eintr(space, e):
                     continue
                 raise
-            if not space.isinstance_w(w_read, space.w_str):
+            if not space.isinstance_w(w_read, space.w_bytes):
                 raise oefmt(space.w_IOError,
                             "peek() should have returned a bytes object, not "
                             "'%T'", w_read)
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -299,7 +299,7 @@
             raise
         return space.newtext('ascii')
     else:
-        if space.isinstance_w(w_encoding, space.w_unicode):
+        if space.isinstance_w(w_encoding, space.w_text):
             return w_encoding
 
     raise oefmt(space.w_IOError, "could not determine default encoding")
@@ -391,9 +391,8 @@
         else:
             newline = space.unicode_w(w_newline)
         if newline and newline not in (u'\n', u'\r\n', u'\r'):
-            r = space.str_w(space.repr(w_newline))
             raise oefmt(space.w_ValueError,
-                        "illegal newline value: %s", r)
+                        "illegal newline value: %R", w_newline)
 
         self.line_buffering = line_buffering
         self.write_through = write_through
@@ -596,7 +595,7 @@
                                     "read1" if self.has_read1 else "read",
                                     space.newint(self.chunk_size))
 
-        if not space.isinstance_w(w_input, space.w_str):
+        if not space.isinstance_w(w_input, space.w_bytes):
             msg = "decoder getstate() should have returned a bytes " \
                   "object not '%T'"
             raise oefmt(space.w_TypeError, msg, w_input)
@@ -908,9 +907,8 @@
                         whence)
 
         if space.is_true(space.lt(w_pos, space.newint(0))):
-            r = space.str_w(space.repr(w_pos))
             raise oefmt(space.w_ValueError,
-                        "negative seek position %s", r)
+                        "negative seek position %R", w_pos)
 
         space.call_method(self, "flush")
 
@@ -932,7 +930,7 @@
             # Just like _read_chunk, feed the decoder and save a snapshot.
             w_chunk = space.call_method(self.w_buffer, "read",
                                         space.newint(cookie.bytes_to_feed))
-            if not space.isinstance_w(w_chunk, space.w_str):
+            if not space.isinstance_w(w_chunk, space.w_bytes):
                 msg = "underlying read() should have returned " \
                       "a bytes object, not '%T'"
                 raise oefmt(space.w_TypeError, msg, w_chunk)
diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -110,7 +110,7 @@
 _strxfrm = rlocale.external('strxfrm',
                     [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T)
 
- at unwrap_spec(s=str)
+ at unwrap_spec(s='text')
 def strxfrm(space, s):
     "string -> string. Returns a string that behaves for cmp locale-aware."
     n1 = len(s) + 1
@@ -155,7 +155,7 @@
 if rlocale.HAVE_LIBINTL:
     _gettext = rlocale.external('gettext', [rffi.CCHARP], rffi.CCHARP)
 
-    @unwrap_spec(msg=str)
+    @unwrap_spec(msg='text')
     def gettext(space, msg):
         """gettext(msg) -> string
         Return translation of msg."""
@@ -167,7 +167,7 @@
 
     _dgettext = rlocale.external('dgettext', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP)
 
-    @unwrap_spec(msg=str)
+    @unwrap_spec(msg='text')
     def dgettext(space, w_domain, msg):
         """dgettext(domain, msg) -> string
         Return translation of msg in domain."""
@@ -201,7 +201,7 @@
     _dcgettext = rlocale.external('dcgettext', [rffi.CCHARP, rffi.CCHARP, rffi.INT],
                                                                 rffi.CCHARP)
 
-    @unwrap_spec(msg=str, category=int)
+    @unwrap_spec(msg='text', category=int)
     def dcgettext(space, w_domain, msg, category):
         """dcgettext(domain, msg, category) -> string
         Return translation of msg in domain and category."""
@@ -263,7 +263,7 @@
                                                                 rffi.CCHARP,
                                        save_err=rffi.RFFI_SAVE_ERRNO)
 
-    @unwrap_spec(domain=str)
+    @unwrap_spec(domain='text')
     def bindtextdomain(space, domain, w_dir):
         """bindtextdomain(domain, dir) -> string
         Bind the C library's domain to dir."""
@@ -294,7 +294,7 @@
                                     [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP)
 
     if rlocale.HAVE_BIND_TEXTDOMAIN_CODESET:
-        @unwrap_spec(domain=str)
+        @unwrap_spec(domain='text')
         def bind_textdomain_codeset(space, domain, w_codeset):
             """bind_textdomain_codeset(domain, codeset) -> string
             Bind the C library's domain to codeset."""
diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py
--- a/pypy/module/_minimal_curses/interp_curses.py
+++ b/pypy/module/_minimal_curses/interp_curses.py
@@ -75,7 +75,7 @@
     except _curses.error as e:
         raise curses_error(e.args[0])
 
- at unwrap_spec(capname=str)
+ at unwrap_spec(capname='text')
 def tigetstr(space, capname):
     try:
         result = _curses_tigetstr(capname)
diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py
--- a/pypy/module/_multibytecodec/interp_multibytecodec.py
+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py
@@ -52,7 +52,7 @@
 MultibyteCodec.typedef.acceptable_as_base_class = False
 
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def getcodec(space, name):
     try:
         codec = c_codecs.getcodec(name)
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -525,9 +525,8 @@
     def after_fork(self):
         self.count = 0
 
-    @unwrap_spec(kind=int, maxvalue=int)
-    def rebuild(space, w_cls, w_handle, kind, maxvalue, w_name):
-        name = space.str_or_None_w(w_name)
+    @unwrap_spec(kind=int, maxvalue=int, name='text_or_None')
+    def rebuild(space, w_cls, w_handle, kind, maxvalue, name):
         #
         if sys_platform != 'win32' and name is not None:
             # like CPython, in this case ignore 'w_handle'
diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py
--- a/pypy/module/_pickle_support/maker.py
+++ b/pypy/module/_pickle_support/maker.py
@@ -69,7 +69,7 @@
     new_iter = W_IntRangeIterator(space, current, remaining, step)
     return new_iter
 
- at unwrap_spec(identifier=str)
+ at unwrap_spec(identifier='text')
 def builtin_code(space, identifier):
     from pypy.interpreter import gateway
     try:
@@ -78,7 +78,7 @@
         raise oefmt(space.w_RuntimeError,
                     "cannot unpickle builtin code: %s", identifier)
 
- at unwrap_spec(identifier=str)
+ at unwrap_spec(identifier='text')
 def builtin_function(space, identifier):
     from pypy.interpreter import function
     try:
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -391,9 +391,7 @@
         return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
 
 def loads(space, w_s, w_errorcls=None):
-    if space.isinstance_w(w_s, space.w_bytes):
-        raise oefmt(space.w_TypeError, "Expected string, got %T", w_s)
-    s = space.str_w(w_s)
+    s = space.text_w(w_s)
     decoder = JSONDecoder(space, s)
     try:
         w_res = decoder.decode_any(0)
@@ -405,8 +403,8 @@
     except DecoderError as e:
         if w_errorcls is None:
             w_errorcls = space.w_ValueError
-        w_e = space.call_function(w_errorcls, space.wrap(e.msg), w_s,
-                                  space.wrap(e.pos))
+        w_e = space.call_function(w_errorcls, space.newtext(e.msg), w_s,
+                                  space.newint(e.pos))
         raise OperationError(w_errorcls, w_e)
     finally:
         decoder.close()
diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py
--- a/pypy/module/_pypyjson/interp_encoder.py
+++ b/pypy/module/_pypyjson/interp_encoder.py
@@ -17,8 +17,8 @@
 
 
 def raw_encode_basestring_ascii(space, w_string):
-    if space.isinstance_w(w_string, space.w_str):
-        s = space.str_w(w_string)
+    if space.isinstance_w(w_string, space.w_bytes):
+        s = space.bytes_w(w_string)
         for i in range(len(s)):
             c = s[i]
             if c >= ' ' and c <= '~' and c != '"' and c != '\\':
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -20,9 +20,9 @@
     def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
         argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
             space, w_argtypes, w_restype)
-        if space.isinstance_w(w_name, space.w_unicode):
+        if space.isinstance_w(w_name, space.w_text):
             # XXX: support LoadLibraryW
-            name = space.str_w(w_name)
+            name = space.text_w(w_name)
             try:
                 func = CDLL.cdll.getpointer(name, argtypes, restype,
                                             flags = CDLL.flags)
@@ -52,9 +52,9 @@
             raise oefmt(space.w_TypeError,
                         "function name must be a string or integer")
 else:
-    @unwrap_spec(name=str)
+    @unwrap_spec(name='text')
     def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
             space, w_argtypes, w_restype)
         try:
@@ -287,7 +287,7 @@
     restype = unwrap_ffitype(space, w_restype, allow_void=True)
     return argtypes_w, argtypes, w_restype, restype
 
- at unwrap_spec(addr=r_uint, name=str, flags=int)
+ at unwrap_spec(addr=r_uint, name='text', flags=int)
 def descr_fromaddr(space, w_cls, addr, name, w_argtypes,
                     w_restype, flags=libffi.FUNCFLAG_CDECL):
     argtypes_w, argtypes, w_restype, restype = unpack_argtypes(space,
@@ -331,7 +331,7 @@
     def getfunc(self, space, w_name, w_argtypes, w_restype):
         return _getfunc(space, self, w_name, w_argtypes, w_restype)
 
-    @unwrap_spec(name=str)
+    @unwrap_spec(name='text')
     def getaddressindll(self, space, name):
         try:
             address_as_uint = rffi.cast(lltype.Unsigned,
diff --git a/pypy/module/_rawffi/alt/interp_struct.py b/pypy/module/_rawffi/alt/interp_struct.py
--- a/pypy/module/_rawffi/alt/interp_struct.py
+++ b/pypy/module/_rawffi/alt/interp_struct.py
@@ -21,7 +21,7 @@
     def __repr__(self):
         return '<Field %s %s>' % (self.name, self.w_ffitype.name)
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def descr_new_field(space, w_type, name, w_ffitype):
     w_ffitype = space.interp_w(W_FFIType, w_ffitype)
     return W_Field(name, w_ffitype)
@@ -115,7 +115,7 @@
 
 
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def descr_new_structdescr(space, w_type, name, w_fields=None):
     descr = W__StructDescr(name)
     if not space.is_none(w_fields):
diff --git a/pypy/module/_rawffi/alt/type_converter.py b/pypy/module/_rawffi/alt/type_converter.py
--- a/pypy/module/_rawffi/alt/type_converter.py
+++ b/pypy/module/_rawffi/alt/type_converter.py
@@ -79,11 +79,11 @@
 
     def maybe_handle_char_or_unichar_p(self, w_ffitype, w_obj):
         w_type = jit.promote(self.space.type(w_obj))
-        if w_ffitype.is_char_p() and w_type is self.space.w_str:
-            strval = self.space.str_w(w_obj)
+        if w_ffitype.is_char_p() and w_type is self.space.w_bytes:
+            strval = self.space.bytes_w(w_obj)
             self.handle_char_p(w_ffitype, w_obj, strval)
             return True
-        elif w_ffitype.is_unichar_p() and (w_type is self.space.w_str or
+        elif w_ffitype.is_unichar_p() and (w_type is self.space.w_bytes or
                                            w_type is self.space.w_unicode):
             unicodeval = self.space.unicode_w(w_obj)
             self.handle_unichar_p(w_ffitype, w_obj, unicodeval)
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -79,7 +79,7 @@
         self.argtypes = unpack_argshapes(space, w_args)
         ffiargs = [tp.get_basic_ffi_type() for tp in self.argtypes]
         if not space.is_w(w_result, space.w_None):
-            self.result = space.str_w(w_result)
+            self.result = space.text_w(w_result)
             ffiresult = letter2tp(space, self.result).get_basic_ffi_type()
         else:
             self.result = None
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -101,8 +101,8 @@
 
 def unpack_simple_shape(space, w_shape):
     # 'w_shape' must be either a letter or a tuple (struct, 1).
-    if space.isinstance_w(w_shape, space.w_unicode):
-        letter = space.str_w(w_shape)
+    if space.isinstance_w(w_shape, space.w_text):
+        letter = space.text_w(w_shape)
         return letter2tp(space, letter)
     else:
         w_shapetype, w_length = space.fixedview(w_shape, expected_length=2)
@@ -112,8 +112,8 @@
 def unpack_shape_with_length(space, w_shape):
     # Allow 'w_shape' to be a letter or any (shape, number).
     # The result is always a W_Array.
-    if space.isinstance_w(w_shape, space.w_unicode):
-        letter = space.str_w(w_shape)
+    if space.isinstance_w(w_shape, space.w_text):
+        letter = space.text_w(w_shape)
         return letter2tp(space, letter)
     else:
         w_shapetype, w_length = space.fixedview(w_shape, expected_length=2)
@@ -193,8 +193,8 @@
         else:
             ffi_restype = ffi_type_void
 
-        if space.isinstance_w(w_name, space.w_unicode):
-            name = space.str_w(w_name)
+        if space.isinstance_w(w_name, space.w_text):
+            name = space.text_w(w_name)
 
             try:
                 ptr = self.cdll.getrawpointer(name, ffi_argtypes, ffi_restype,
@@ -225,7 +225,7 @@
         space.setitem(self.w_cache, w_key, w_funcptr)
         return w_funcptr
 
-    @unwrap_spec(name=str)
+    @unwrap_spec(name='text')
     def getaddressindll(self, space, name):
         try:
             address_as_uint = rffi.cast(lltype.Unsigned,
@@ -411,10 +411,10 @@
         if space.isinstance_w(w_arg, space.w_int):
             val = getbytevalue(space, w_arg)
         else:
-            s = space.str_w(w_arg)
+            s = space.bytes_w(w_arg)
             if len(s) != 1:
                 raise oefmt(space.w_TypeError,
-                            "Expected string of length one as character")
+                            "Expected bytes of length one as character")
             val = s[0]
         push_func(add_arg, argdesc, val)
     elif letter == 'u':
@@ -565,7 +565,7 @@
 W_FuncPtr.typedef.acceptable_as_base_class = False
 
 def _create_new_accessor(func_name, name):
-    @unwrap_spec(tp_letter=str)
+    @unwrap_spec(tp_letter='text')
     def accessor(space, tp_letter):
         if len(tp_letter) != 1:
             raise oefmt(space.w_ValueError, "Expecting string of length one")
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -37,7 +37,7 @@
                         "Expected list of 2- or 3-size tuples")
 
         try:
-            name = space.str_w(l_w[0])
+            name = space.text_w(l_w[0])
         except OperationError:
             raise oefmt(space.w_TypeError,
                         "structure field name must be string not %T", l_w[0])
@@ -204,12 +204,12 @@
     def fromaddress(self, space, address):
         return W_StructureInstance(space, self, address)
 
-    @unwrap_spec(attr=str)
+    @unwrap_spec(attr='text')
     def descr_fieldoffset(self, space, attr):
         index = self.getindex(space, attr)
         return space.newint(self.ll_positions[index])
 
-    @unwrap_spec(attr=str)
+    @unwrap_spec(attr='text')
     def descr_fieldsize(self, space, attr):
         index = self.getindex(space, attr)
         if self.ll_bitsizes and index < len(self.ll_bitsizes):
@@ -352,7 +352,7 @@
         addr = rffi.cast(lltype.Unsigned, self.ll_buffer)
         return space.newtext("<_rawffi struct %x>" % (addr,))
 
-    @unwrap_spec(attr=str)
+    @unwrap_spec(attr='text')
     def getattr(self, space, attr):
         if not self.ll_buffer:
             raise segfault_exception(space, "accessing NULL pointer")
@@ -360,7 +360,7 @@
         _, tp, _ = self.shape.fields[i]
         return wrap_value(space, cast_pos, self, i, tp.itemcode)
 
-    @unwrap_spec(attr=str)
+    @unwrap_spec(attr='text')
     def setattr(self, space, attr, w_value):
         if not self.ll_buffer:
             raise segfault_exception(space, "accessing NULL pointer")
@@ -368,7 +368,7 @@
         _, tp, _ = self.shape.fields[i]
         unwrap_value(space, push_field, self, i, tp.itemcode, w_value)
 
-    @unwrap_spec(attr=str)
+    @unwrap_spec(attr='text')
     def descr_fieldaddress(self, space, attr):
         i = self.shape.getindex(space, attr)
         ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -73,7 +73,7 @@
         raise converted_error(space, e)
     return common_wrapgethost(space, res)
 
- at unwrap_spec(name=str, w_proto = WrappedDefault(None))
+ at unwrap_spec(name='text', w_proto = WrappedDefault(None))
 def getservbyname(space, name, w_proto):
     """getservbyname(servicename[, protocolname]) -> integer
 
@@ -84,7 +84,7 @@
     if space.is_w(w_proto, space.w_None):
         proto = None
     else:
-        proto = space.str_w(w_proto)
+        proto = space.text_w(w_proto)
     try:
         port = rsocket.getservbyname(name, proto)
     except SocketError as e:
@@ -102,7 +102,7 @@
     if space.is_w(w_proto, space.w_None):
         proto = None
     else:
-        proto = space.str_w(w_proto)
+        proto = space.text_w(w_proto)
 
     if port < 0 or port > 0xffff:
         raise oefmt(space.w_OverflowError,
@@ -114,7 +114,7 @@
         raise converted_error(space, e)
     return space.newtext(service)
 
- at unwrap_spec(name=str)
+ at unwrap_spec(name='text')
 def getprotobyname(space, name):
     """getprotobyname(name) -> integer
 
@@ -229,7 +229,7 @@
         raise oefmt(space.w_OverflowError, "long int larger than 32 bits")
     return space.newint(rsocket.htonl(r_uint32(x)))
 
- at unwrap_spec(ip=str)
+ at unwrap_spec(ip='text')
 def inet_aton(space, ip):
     """inet_aton(string) -> packed 32-bit IP representation
 
@@ -254,7 +254,7 @@
         raise converted_error(space, e)
     return space.newtext(ip)
 
- at unwrap_spec(family=int, ip=str)
+ at unwrap_spec(family=int, ip='text')
 def inet_pton(space, family, ip):
     """inet_pton(family, ip) -> packed IP address string
 
@@ -267,7 +267,7 @@
         raise converted_error(space, e)
     return space.newbytes(buf)
 
- at unwrap_spec(family=int, packed="bufferstr")
+ at unwrap_spec(family=int, packed='bufferstr')
 def inet_ntop(space, family, packed):
     """inet_ntop(family, packed_ip) -> string formatted IP address
 
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -149,14 +149,14 @@
             raise oefmt(space.w_TypeError,
                         "AF_PACKET address must be a tuple of length 2 "
                         "to 5, not %d", len(pieces_w))
-        ifname = space.str_w(pieces_w[0])
+        ifname = space.text_w(pieces_w[0])
         ifindex = rsocket.PacketAddress.get_ifindex_from_ifname(fd, ifname)
         protocol = space.int_w(pieces_w[1])
         if len(pieces_w) > 2: pkttype = space.int_w(pieces_w[2])
         else:                 pkttype = 0
         if len(pieces_w) > 3: hatype = space.int_w(pieces_w[3])
         else:                 hatype = 0
-        if len(pieces_w) > 4: haddr = space.str_w(pieces_w[4])
+        if len(pieces_w) > 4: haddr = space.text_w(pieces_w[4])
         else:                 haddr = ""
         if len(haddr) > 8:
             raise oefmt(space.w_ValueError,
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -329,7 +329,7 @@
                     filter_as_string = buf.as_str()
                 literal = '\\' not in filter_as_string
                 use_builder = (
-                    space.isinstance_w(w_string, space.w_str) and literal)
+                    space.isinstance_w(w_string, space.w_bytes) and literal)
             if literal:
                 w_filter = w_ptemplate
                 filter_is_callable = False
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -291,7 +291,7 @@
         w_text = space.str(w_message)
         w_category = space.type(w_message)
     elif (not space.isinstance_w(w_message, space.w_unicode) or
-          not space.isinstance_w(w_message, space.w_str)):
+          not space.isinstance_w(w_message, space.w_bytes)):
         w_text = space.str(w_message)
         w_message = space.call_function(w_category, w_message)
     else:
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -149,7 +149,7 @@
         if ret != 0:
             raiseWindowsError(space, ret, 'RegFlushKey')
 
- at unwrap_spec(subkey=str, filename=str)
+ at unwrap_spec(subkey="text", filename="text")
 def LoadKey(space, w_hkey, subkey, filename):
     """LoadKey(key, sub_key, file_name) - Creates a subkey under the specified key
 and stores registration information from a specified file into that subkey.
@@ -174,7 +174,7 @@
     if ret != 0:
         raiseWindowsError(space, ret, 'RegLoadKey')
 
- at unwrap_spec(filename=str)
+ at unwrap_spec(filename="text")
 def SaveKey(space, w_hkey, filename):
     """SaveKey(key, file_name) - Saves the specified key, and all its subkeys to the specified file.
 
@@ -193,7 +193,7 @@
     if ret != 0:
         raiseWindowsError(space, ret, 'RegSaveKey')
 
- at unwrap_spec(typ=int, value=str)
+ at unwrap_spec(typ=int, value="text")
 def SetValue(space, w_hkey, w_subkey, typ, value):
     """SetValue(key, sub_key, type, value) - Associates a value with a specified key.
 
@@ -218,7 +218,7 @@
     if space.is_w(w_subkey, space.w_None):
         subkey = None
     else:
-        subkey = space.str_w(w_subkey)
+        subkey = space.text_w(w_subkey)
     with rffi.scoped_str2charp(value) as dataptr:
         ret = rwinreg.RegSetValue(hkey, subkey, rwinreg.REG_SZ, dataptr, len(value))
         if ret != 0:
@@ -239,7 +239,7 @@
     if space.is_w(w_subkey, space.w_None):
         subkey = None
     else:
-        subkey = space.str_w(w_subkey)
+        subkey = space.text_w(w_subkey)
     with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p:
         ret = rwinreg.RegQueryValue(hkey, subkey, None, bufsize_p)
         bufSize = intmask(bufsize_p[0])
@@ -285,7 +285,7 @@
             if space.isinstance_w(w_value, space.w_unicode):
                 w_value = space.call_method(w_value, 'encode',
                                             space.newtext('mbcs'))
-            buf = rffi.str2charp(space.str_w(w_value))
+            buf = rffi.str2charp(space.text_w(w_value))
             buflen = space.len_w(w_value) + 1
 
     elif typ == rwinreg.REG_MULTI_SZ:
@@ -305,7 +305,7 @@
                     if space.isinstance_w(w_item, space.w_unicode):
                         w_item = space.call_method(w_item, 'encode',
                                                    space.newtext('mbcs'))
-                    item = space.str_w(w_item)
+                    item = space.bytes_w(w_item)
                     strings.append(item)
                     buflen += len(item) + 1
                 except OperationError as e:
@@ -387,7 +387,7 @@
     else: # REG_BINARY and all other types
         return space.newbytes(rffi.charpsize2str(buf, buflen))
 
- at unwrap_spec(value_name=str, typ=int)
+ at unwrap_spec(value_name="text", typ=int)
 def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value):
     """SetValueEx(key, value_name, reserved, type, value) - Stores data in the value field of an open registry key.
 
@@ -437,7 +437,7 @@
     if space.is_w(w_subkey, space.w_None):
         subkey = None
     else:
-        subkey = space.str_w(w_subkey)

From pypy.commits at gmail.com  Wed Feb 15 07:34:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 04:34:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: CPython 3.x accepts both bytes or
 unicodes as the format specification of the struct module
Message-ID: <58a44ad0.7388df0a.d39f7.9144@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90148:2d5c0f6653b6
Date: 2017-02-15 13:33 +0100
http://bitbucket.org/pypy/pypy/changeset/2d5c0f6653b6/

Log:	CPython 3.x accepts both bytes or unicodes as the format
	specification of the struct module

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -32,8 +32,15 @@
     return fmtiter.totalsize
 
 
- at unwrap_spec(format='text')
-def calcsize(space, format):
+def text_or_bytes_w(space, w_input):
+    # why does CPython do this??
+    if space.isinstance_w(w_input, space.w_bytes):
+        return space.bytes_w(w_input)
+    else:
+        return space.text_w(w_input)
+
+def calcsize(space, w_format):
+    format = text_or_bytes_w(space, w_format)
     return space.newint(_calcsize(space, format))
 
 
@@ -52,14 +59,21 @@
     return fmtiter.result.build()
 
 
- at unwrap_spec(format='text')
-def pack(space, format, args_w):
+def pack(space, w_format, args_w):
+    format = text_or_bytes_w(space, w_format)
+    return do_pack(space, format, args_w)
+
+def do_pack(space, format, args_w):
     return space.newbytes(_pack(space, format, args_w))
 
 
+ at unwrap_spec(offset=int)
+def pack_into(space, w_format, w_buffer, offset, args_w):
+    format = text_or_bytes_w(space, w_format)
+    return do_pack_into(space, format, w_buffer, offset, args_w)
+
 # XXX inefficient
- at unwrap_spec(format='text', offset=int)
-def pack_into(space, format, w_buffer, offset, args_w):
+def do_pack_into(space, format, w_buffer, offset, args_w):
     res = _pack(space, format, args_w)
     buf = space.getarg_w('w*', w_buffer)
     if offset < 0:
@@ -83,14 +97,21 @@
     return space.newtuple(fmtiter.result_w[:])
 
 
- at unwrap_spec(format='text')
-def unpack(space, format, w_str):
+def unpack(space, w_format, w_str):
+    format = text_or_bytes_w(space, w_format)
+    return do_unpack(space, format, w_str)
+
+def do_unpack(space, format, w_str):
     buf = space.getarg_w('s*', w_str)
     return _unpack(space, format, buf)
 
 
- at unwrap_spec(format='text', offset=int)
-def unpack_from(space, format, w_buffer, offset=0):
+ at unwrap_spec(offset=int)
+def unpack_from(space, w_format, w_buffer, offset=0):
+    format = text_or_bytes_w(space, w_format)
+    return do_unpack_from(space, format, w_buffer, offset)
+
+def do_unpack_from(space, format, w_buffer, offset=0):
     size = _calcsize(space, format)
     buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
     if offset < 0:
@@ -146,25 +167,25 @@
         self.format = format
         self.size = _calcsize(space, format)
 
-    @unwrap_spec(format='text')
-    def descr__new__(space, w_subtype, format):
+    def descr__new__(space, w_subtype, w_format):
+        format = text_or_bytes_w(space, w_format)
         self = space.allocate_instance(W_Struct, w_subtype)
         W_Struct.__init__(self, space, format)
         return self
 
     def descr_pack(self, space, args_w):
-        return pack(space, jit.promote_string(self.format), args_w)
+        return do_pack(space, jit.promote_string(self.format), args_w)
 
     @unwrap_spec(offset=int)
     def descr_pack_into(self, space, w_buffer, offset, args_w):
-        return pack_into(space, jit.promote_string(self.format), w_buffer, offset, args_w)
+        return do_pack_into(space, jit.promote_string(self.format), w_buffer, offset, args_w)
 
     def descr_unpack(self, space, w_str):
-        return unpack(space, jit.promote_string(self.format), w_str)
+        return do_unpack(space, jit.promote_string(self.format), w_str)
 
     @unwrap_spec(offset=int)
     def descr_unpack_from(self, space, w_buffer, offset=0):
-        return unpack_from(space, jit.promote_string(self.format), w_buffer, offset)
+        return do_unpack_from(space, jit.promote_string(self.format), w_buffer, offset)
 
     def descr_iter_unpack(self, space, w_buffer):
         return W_UnpackIter(space, self, w_buffer)
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -530,6 +530,16 @@
                 format = byteorder+code
                 t = run_not_int_test(format)
 
+    def test_struct_with_bytes_as_format_string(self):
+        # why??
+        assert self.struct.calcsize(b'!ii') == 8
+        b = memoryview(bytearray(8))
+        self.struct.iter_unpack(b'ii', b)
+        self.struct.pack(b"ii", 45, 56)
+        self.struct.pack_into(b"ii", b, 0, 45, 56)
+        self.struct.unpack(b"ii", b"X" * 8)
+        assert self.struct.unpack_from(b"ii", b) == (45, 56)
+
 
 class AppTestStructBuffer(object):
     spaceconfig = dict(usemodules=['struct', '__pypy__'])
@@ -561,6 +571,7 @@
         assert self.struct.unpack_from("ii", b, 2) == (17, 42)
         b[:sz] = self.struct.pack("ii", 18, 43)
         assert self.struct.unpack_from("ii", b) == (18, 43)
+        self.struct.Struct(b"ii")
 
 
 class AppTestFastPath(object):

From pypy.commits at gmail.com  Wed Feb 15 07:36:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 04:36:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: move this line at the right place
Message-ID: <58a44b62.69acdf0a.29c3f.220a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90149:11c131a2deee
Date: 2017-02-15 13:36 +0100
http://bitbucket.org/pypy/pypy/changeset/11c131a2deee/

Log:	move this line at the right place

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -539,6 +539,7 @@
         self.struct.pack_into(b"ii", b, 0, 45, 56)
         self.struct.unpack(b"ii", b"X" * 8)
         assert self.struct.unpack_from(b"ii", b) == (45, 56)
+        self.struct.Struct(b"ii")
 
 
 class AppTestStructBuffer(object):
@@ -571,7 +572,6 @@
         assert self.struct.unpack_from("ii", b, 2) == (17, 42)
         b[:sz] = self.struct.pack("ii", 18, 43)
         assert self.struct.unpack_from("ii", b) == (18, 43)
-        self.struct.Struct(b"ii")
 
 
 class AppTestFastPath(object):

From pypy.commits at gmail.com  Wed Feb 15 08:18:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 05:18:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge d8f5f0260d45
Message-ID: <58a45537.41a3df0a.887f3.8f9f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90150:be103543e654
Date: 2017-02-15 13:51 +0100
http://bitbucket.org/pypy/pypy/changeset/be103543e654/

Log:	hg merge d8f5f0260d45


From pypy.commits at gmail.com  Wed Feb 15 08:18:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 05:18:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: improve space.newlist_text()
Message-ID: <58a45539.52a5df0a.1b8dd.95b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90151:966d42ca0e4e
Date: 2017-02-15 13:57 +0100
http://bitbucket.org/pypy/pypy/changeset/966d42ca0e4e/

Log:	improve space.newlist_text()

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -115,9 +115,8 @@
             return w_dict.getitem(w_key)
 
     def w_keys(self, w_dict):
-        space = self.space
-        keys = self.unerase(w_dict.dstorage)[0]
-        return space.newlist_unicode([key.decode('utf-8') for key in keys])
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_text(l[:])
 
     def values(self, w_dict):
         return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -304,8 +304,8 @@
         return W_ListObject.newlist_bytes(self, list_s)
 
     def newlist_text(self, list_t):
-        # XXX improve!
-        return self.newlist([self.newtext(t) for t in list_t])
+        return self.newlist_unicode([
+            decode_utf8(self, s, allow_surrogates=True) for s in list_t])
 
     def newlist_unicode(self, list_u):
         return W_ListObject.newlist_unicode(self, list_u)
@@ -356,9 +356,6 @@
     def newtext(self, s):
         return self.newunicode(decode_utf8(self, s, allow_surrogates=True))
 
-    def newtext(self, s):
-        return self.newunicode(decode_utf8(self, s, allow_surrogates=True))
-
     def newtext_or_none(self, s):
         if s is None:
             return self.w_None

From pypy.commits at gmail.com  Wed Feb 15 08:18:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 05:18:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge 7ae20dbc22fb
Message-ID: <58a4553c.4395df0a.33d52.927e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90152:d80fad7c1292
Date: 2017-02-15 14:18 +0100
http://bitbucket.org/pypy/pypy/changeset/d80fad7c1292/

Log:	hg merge 7ae20dbc22fb

	(note: I didn't review the very large number of conflicts, because I
	*think* that in this exact case they should all resolve to nothing.
	Indeed we're merging 7ae20dbc22fb, which is itself the merge of two
	changesets that are already in my part history)


From pypy.commits at gmail.com  Wed Feb 15 10:41:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 07:41:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge space-newtext (yay)
Message-ID: <58a476a0.90b1df0a.392a5.b6e6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90153:c9c2a8f5f76b
Date: 2017-02-15 14:31 +0100
http://bitbucket.org/pypy/pypy/changeset/c9c2a8f5f76b/

Log:	hg merge space-newtext (yay)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1571,7 +1571,7 @@
     def text_or_None_w(self, w_obj):
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
-    @not_rpython
+    #@not_rpython    BACKCOMPAT: should be replaced with bytes_w or text_w
     def str_w(self, w_obj):
         """
         if w_obj is unicode, call text_w() (i.e., return the UTF-8-nosg
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1129,6 +1129,8 @@
             assert space._code_of_sys_exc_info is None
             space._code_of_sys_exc_info = code
         #
+        if hasattr(space, '_see_interp2app'):
+            space._see_interp2app(gateway)      # only for fake/objspace.py
         return fn
 
 
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -359,6 +359,11 @@
         raise oefmt(space.w_AttributeError,
                     "generic property has no __objclass__")
 
+    def spacebind(self, space):
+        if hasattr(space, '_see_getsetproperty'):
+            space._see_getsetproperty(self)      # only for fake/objspace.py
+        return self
+
 
 def interp_attrproperty(name, cls, doc=None, wrapfn=None):
     "NOT_RPYTHON: initialization-time only"
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -188,7 +188,6 @@
             guard_no_exception(descr=...)
             p95 = call_r(..., descr=<Callr . r EF=5>)     # ll_build
             guard_no_exception(descr=...)
-            guard_nonnull(p95, descr=...)
             i96 = strlen(p95)
             i97 = int_add_ovf(i71, i96)
             guard_no_overflow(descr=...)
@@ -273,7 +272,6 @@
         guard_not_invalidated(descr=...)
         p53 = call_r(ConstClass(fast_str_decode_ascii), p80, descr=<Callr . r EF=4>)
         guard_no_exception(descr=...)
-        guard_nonnull(p53, descr=...)
         --TICK--
         jump(..., descr=...)
         """)
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -264,6 +264,7 @@
         assert pack(">?", False) == b'\x00'
         assert pack("@?", True) == b'\x01'
         assert pack("@?", False) == b'\x00'
+        assert self.struct.unpack("?", 'X')[0] is True
 
     def test_transitiveness(self):
         c = b'a'
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -58,7 +58,10 @@
 
     def bigint_w(self, space, allow_conversion=True):
         from rpython.rlib.rbigint import rbigint
-        return rbigint.fromint(NonConstant(42))
+        x = 42
+        if we_are_translated():
+            x = NonConstant(x)
+        return rbigint.fromint(x)
 
 class W_MyListObj(W_MyObject):
     def append(self, w_other):
@@ -123,6 +126,9 @@
                  'unicode', 'complex', 'slice', 'bool', 'text', 'object',
                  'set', 'frozenset', 'bytearray', 'memoryview']
 
+INTERP_TYPES = ['function', 'builtin_function', 'module', 'getset_descriptor',
+                'instance', 'classobj']
+
 class FakeObjSpace(ObjSpace):
     is_fake_objspace = True
 
@@ -220,10 +226,8 @@
     @not_rpython
     def wrap(self, x):
         if not we_are_translated():
-            if isinstance(x, gateway.interp2app):
-                self._see_interp2app(x)
-            if isinstance(x, GetSetProperty):
-                self._see_getsetproperty(x)
+            if isinstance(x, W_Root):
+                x.spacebind(self)
         if isinstance(x, r_singlefloat):
             self._wrap_not_rpython(x)
         if isinstance(x, list):
@@ -234,6 +238,7 @@
 
     @not_rpython
     def _see_interp2app(self, interp2app):
+        """Called by GatewayCache.build()"""
         activation = interp2app._code.activation
         def check():
             scope_w = [w_some_obj()] * NonConstant(42)
@@ -244,6 +249,7 @@
 
     @not_rpython
     def _see_getsetproperty(self, getsetproperty):
+        """Called by GetSetProperty.spacebind()"""
         space = self
         def checkprop():
             getsetproperty.fget(getsetproperty, space, w_some_obj())
@@ -376,6 +382,7 @@
         ann = t.buildannotator()
         def _do_startup():
             self.threadlocals.enter_thread(self)
+            W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj())
         ann.build_types(_do_startup, [], complete_now=False)
         if func is not None:
             ann.build_types(func, argtypes, complete_now=False)
@@ -436,7 +443,9 @@
 @specialize.memo()
 def see_typedef(space, typedef):
     assert isinstance(typedef, TypeDef)
-    if typedef.name not in BUILTIN_TYPES:
+    if typedef.name not in BUILTIN_TYPES and typedef.name not in INTERP_TYPES:
+        print
+        print '------ seeing typedef %r ------' % (typedef.name,)
         for name, value in typedef.rawdict.items():
             space.wrap(value)
 
diff --git a/pypy/objspace/fake/test/test_checkmodule.py b/pypy/objspace/fake/test/test_checkmodule.py
--- a/pypy/objspace/fake/test/test_checkmodule.py
+++ b/pypy/objspace/fake/test/test_checkmodule.py
@@ -19,7 +19,7 @@
     space = FakeObjSpace()
     assert len(space._seen_extras) == 1
     assert len(check) == 0
-    space.wrap(interp2app(lambda space: see()))
+    interp2app(lambda space: see()).spacebind(space)
     assert len(space._seen_extras) == 2
     assert len(check) == 0
     space.translates()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -135,6 +135,11 @@
         assert typedef is not None
         return self.fromcache(TypeCache).getorbuild(typedef)
 
+    # BACKCOMPAT: this function is still accepted for backward
+    # compatibility, but its usage should be progressively removed
+    # everywhere apart from tests.
+    #@not_rpython # only for tests
+    @specialize.argtype(1)
     def wrap(self, x):
         """ Wraps the Python value 'x' into one of the wrapper classes. This
         should only be used for tests, in real code you need to use the
@@ -183,6 +188,12 @@
         if is_valid_int(x):
             return self.newint(x)
 
+        return self._wrap_not_rpython(x)
+
+    def _wrap_not_rpython(self, x):
+        "NOT_RPYTHON"
+        # _____ this code is here to support testing only _____
+
         # wrap() of a container works on CPython, but the code is
         # not RPython.  Don't use -- it is kept around mostly for tests.
         # Use instead newdict(), newlist(), newtuple().
diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -74,6 +74,7 @@
         for name in ('int', 'long', 'str', 'unicode', 'list', 'None', 'ValueError',
                 'OverflowError'):
             setattr(self, 'w_' + name, eval(name))
+        self.w_bytes = bytes
         import __builtin__ as __builtin__
         self.builtin = __builtin__
 

From pypy.commits at gmail.com  Wed Feb 15 10:41:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 07:41:23 -0800 (PST)
Subject: [pypy-commit] pypy default: fix command (thanks jason on irc)
Message-ID: <58a476a3.d5941c0a.a0af.e0c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90154:393ff835a097
Date: 2017-02-15 16:40 +0100
http://bitbucket.org/pypy/pypy/changeset/393ff835a097/

Log:	fix command (thanks jason on irc)

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -159,7 +159,8 @@
 import libraries in the `out-of-line API mode`_. This is done by the following
 command::
 
-   PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py
+   cd pypy/goal
+   PYTHONPATH=../.. ./pypy-c ../tool/build_cffi_imports.py
 
 .. _`out-of-line API mode`: http://cffi.readthedocs.org/en/latest/overview.html#real-example-api-level-out-of-line
 

From pypy.commits at gmail.com  Wed Feb 15 10:41:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 07:41:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: fixes
Message-ID: <58a476c6.ad8ddf0a.3a8f1.bb19@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90155:9bf4d7af2fe4
Date: 2017-02-15 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/9bf4d7af2fe4/

Log:	fixes

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1066,7 +1066,8 @@
                     if w_def is Ellipsis:
                         if isinstance(defaultval, str) and (
                                 # XXX hackish
-                           spec == 'bytes' or isinstance(spec, WrappedDefault)):
+                                spec == 'bytes' or spec == 'bufferstr'
+                                or isinstance(spec, WrappedDefault)):
                             w_def = space.newbytes(defaultval)
                         else:
                             w_def = space.wrap(defaultval)
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -120,7 +120,7 @@
         w_EnvironmentError = [EnvironmentError]
         def wrap(self, obj):
             return [obj]
-        newint = newtext = wrap
+        newint = newtext = newunicode = wrap_fsdecoded = wrap
         def call_function(self, exc, w_errno, w_msg, w_filename=None):
             return (exc, w_errno, w_msg, w_filename)
     space = FakeSpace()

From pypy.commits at gmail.com  Wed Feb 15 10:47:33 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 15 Feb 2017 07:47:33 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: add s390x to the enumeration in
 one of the first download paragraphs
Message-ID: <58a47815.c3a4df0a.d49c9.c012@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r863:b395a4d19079
Date: 2017-02-15 16:47 +0100
http://bitbucket.org/pypy/pypy.org/changeset/b395a4d19079/

Log:	add s390x to the enumeration in one of the first download paragraphs

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,8 @@
 <p class="download-menu">There are <a class="reference external" href="http://buildbot.pypy.org/nightly/trunk/">nightly binary builds</a> available. Those builds are not always
 as stable as the release, but they contain numerous bugfixes and
 performance improvements.</p>
-<p>We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:</p>
+<p>We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
+Linux, Mac OS/X and Windows:</p>
 <ul class="simple">
 <li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.6.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html">what's new in PyPy2.7?</a>)</li>
 <li>the Python3.3 compatible release &#8212; <strong>PyPy3.3 v5.5</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3.3-v5.5.0.html">what's new in PyPy3.3?</a>).</li>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -12,7 +12,8 @@
   as stable as the release, but they contain numerous bugfixes and
   performance improvements.
 
-We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
+We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
+Linux, Mac OS/X and Windows:
 
 * the Python2.7 compatible release &#8212; **PyPy2.7 v5.6.0** &#8212; (`what's new in PyPy2.7?`_)
 

From pypy.commits at gmail.com  Wed Feb 15 13:14:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 10:14:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: Some remaining space.wrap(),
 found by running test_ztranslation
Message-ID: <58a49a69.b6a9df0a.a5c35.ede9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90156:426af1de6a1b
Date: 2017-02-15 17:59 +0100
http://bitbucket.org/pypy/pypy/changeset/426af1de6a1b/

Log:	Some remaining space.wrap(), found by running test_ztranslation

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -102,7 +102,7 @@
         self.co_code = code
         self.co_consts_w = consts
         self.co_names_w = [
-            space.new_interned_w_str(space.wrap(aname.decode('utf-8')))
+            space.new_interned_w_str(space.newtext(aname))
             for aname in names]
         self.co_varnames = varnames
         self.co_freevars = freevars
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -530,7 +530,7 @@
     return space.newint(code.signature().num_argnames())
 
 def fget_zero(space, code):
-    return space.wrap(0)
+    return space.newint(0)
 
 def fget_co_flags(space, code): # unwrapping through unwrap_spec
     sig = code.signature()
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -30,11 +30,11 @@
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeEncodeError,
-                             space.newtuple([space.wrap(encoding),
-                                             space.wrap(u),
-                                             space.wrap(startingpos),
-                                             space.wrap(endingpos),
-                                             space.wrap(msg)]))
+                             space.newtuple([space.newtext(encoding),
+                                             space.newunicode(u),
+                                             space.newint(startingpos),
+                                             space.newint(endingpos),
+                                             space.newtext(msg)]))
     return raise_unicode_exception_encode
 
 class RUnicodeEncodeError(Exception):
@@ -82,8 +82,8 @@
         from pypy.module.sys.interp_encoding import getfilesystemencoding
         return space.call_method(w_string, 'decode',
                                  getfilesystemencoding(space),
-                                 space.wrap('surrogateescape'))
-    return space.wrap(uni)
+                                 space.newtext('surrogateescape'))
+    return space.newunicode(uni)
 
 def fsencode(space, w_uni):
     state = space.fromcache(interp_codecs.CodecState)
@@ -113,7 +113,7 @@
         from pypy.module.sys.interp_encoding import getfilesystemencoding
         return space.call_method(w_uni, 'encode',
                                  getfilesystemencoding(space),
-                                 space.wrap('surrogateescape'))
+                                 space.newtext('surrogateescape'))
     return space.newbytes(bytes)
 
 def encode(space, w_data, encoding=None, errors='strict'):
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -50,7 +50,7 @@
         return self.buffer
 
     def descr_len(self, space):
-        return space.wrap(self.buffer.getlength())
+        return space.newint(self.buffer.getlength())
 
     def descr_getitem(self, space, w_index):
         start, stop, step, size = space.decode_index4(w_index,
diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -58,7 +58,7 @@
                 errorpos = rffi.cast(lltype.Signed, errorposp[0])
                 if errorpos == -1:
                     raise MemoryError
-                errmsg = _errmsg(u"pypy_wchar2char")
+                errmsg = _errmsg("pypy_wchar2char")
                 errorhandler('strict', 'filesystemencoding', errmsg, u,
                              errorpos, errorpos + 1)
             return rffi.charp2str(sbuf)
@@ -81,7 +81,7 @@
             ubuf = pypy_char2wchar(sbuf, sizep)
         try:
             if not ubuf:
-                errmsg = _errmsg(u"pypy_char2wchar")
+                errmsg = _errmsg("pypy_char2wchar")
                 errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
             size = rffi.cast(lltype.Signed, sizep[0])
             return rawwcharp2unicoden(ubuf, size)
@@ -91,7 +91,7 @@
 
 def _errmsg(what):
     # I *think* that the functions in locale_codec.c don't set errno
-    return u"%s failed" % what
+    return "%s failed" % what
 
 
 class scoped_unicode2rawwcharp:
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -59,7 +59,7 @@
     def descr_new(space, w_subtype, __args__):
         self = space.allocate_instance(W_BytesIO, w_subtype)
         W_BytesIO.__init__(self, space)
-        return space.wrap(self)
+        return self
 
     def descr_init(self, space, w_initial_bytes=None):
         self.init()
@@ -125,7 +125,7 @@
 
     def getbuffer_w(self, space):
         self._check_closed(space)
-        return space.wrap(W_MemoryView(BytesIOBuffer(self)))
+        return W_MemoryView(BytesIOBuffer(self))
 
     def getvalue_w(self, space):
         self._check_closed(space)
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -23,7 +23,7 @@
         W_BufferedWriter, W_BufferedReader)
 
     if not (space.isinstance_w(w_file, space.w_unicode) or
-            space.isinstance_w(w_file, space.w_str) or
+            space.isinstance_w(w_file, space.w_bytes) or
             space.isinstance_w(w_file, space.w_int)):
         raise oefmt(space.w_TypeError, "invalid file: %R", w_file)
 
@@ -88,7 +88,7 @@
                     "binary mode doesn't take a newline argument")
     w_raw = space.call_function(
         space.gettypefor(W_FileIO), w_file, space.newtext(rawmode),
-        space.newbool(closefd), w_opener)
+        space.newbool(bool(closefd)), w_opener)
 
     isatty = space.is_true(space.call_method(w_raw, "isatty"))
     line_buffering = buffering == 1 or (buffering < 0 and isatty)
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -94,7 +94,7 @@
             # the object is probably in an unusable state, so ignore.
             if w_closed is not None and not space.is_true(w_closed):
                 try:
-                    self._dealloc_warn_w(space, space.wrap(self))
+                    self._dealloc_warn_w(space, self)
                 finally:
                     space.call_method(self, "close")
         except OperationError:
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -461,7 +461,7 @@
 
     def handle_get(self, space):
         h = rffi.cast(rffi.INTPTR_T, self.handle)
-        return space.wrap(h)
+        return space.newint(h)
 
     def get_count(self, space):
         return space.newint(self.count)
diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -220,7 +220,7 @@
         if l_fds_to_keep:
             lltype.free(l_fds_to_keep, flavor='raw')
 
-    return space.wrap(pid)
+    return space.newint(pid)
 
 
 def cloexec_pipe(space):
@@ -233,6 +233,6 @@
         if res != 0:
             raise exception_from_saved_errno(space, space.w_OSError)
 
-        return space.newtuple([space.wrap(fds[0]),
-                               space.wrap(fds[1]),
+        return space.newtuple([space.newint(fds[0]),
+                               space.newint(fds[1]),
                                ])
diff --git a/pypy/module/array/reconstructor.py b/pypy/module/array/reconstructor.py
--- a/pypy/module/array/reconstructor.py
+++ b/pypy/module/array/reconstructor.py
@@ -98,7 +98,7 @@
         memstr = space.bytes_w(w_items)
         step = descr.bytes
         converted_items = [
-            space.wrap(ieee.unpack_float(
+            space.newfloat(ieee.unpack_float(
                     memstr[i:i+step],
                     descr.big_endian))
             for i in range(0, len(memstr), step)]
@@ -106,16 +106,16 @@
 
     elif mformat_code == UTF16_LE:
         w_converted_items = space.call_method(
-            w_items, "decode", space.wrap("utf-16-le"))
+            w_items, "decode", space.newtext("utf-16-le"))
     elif mformat_code == UTF16_BE:
         w_converted_items = space.call_method(
-            w_items, "decode", space.wrap("utf-16-be"))
+            w_items, "decode", space.newtext("utf-16-be"))
     elif mformat_code == UTF32_LE:
         w_converted_items = space.call_method(
-            w_items, "decode", space.wrap("utf-32-le"))
+            w_items, "decode", space.newtext("utf-32-le"))
     elif mformat_code == UTF32_BE:
         w_converted_items = space.call_method(
-            w_items, "decode", space.wrap("utf-32-be"))
+            w_items, "decode", space.newtext("utf-32-be"))
     else:
         descr = format_descriptors[mformat_code]
         # If possible, try to pack array's items using a data type
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -88,10 +88,10 @@
             source = fp.read()
         pathname = "<frozen %s>" % modulename
         code_w = ec.compiler.compile(source, pathname, 'exec', 0)
-        w_mod = add_module(space, space.wrap(modulename))
+        w_mod = add_module(space, space.newtext(modulename))
         assert isinstance(w_mod, Module) # XXX why is that necessary?
         space.setitem(space.sys.get('modules'), w_mod.w_name, w_mod)
-        space.setitem(w_mod.w_dict, space.wrap('__name__'), w_mod.w_name)
+        space.setitem(w_mod.w_dict, space.newtext('__name__'), w_mod.w_name)
         code_w.exec_code(space, w_mod.w_dict, w_mod.w_dict)
         assert check_sys_modules_w(space, modulename)
     finally:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -783,7 +783,7 @@
 def strerror(space, code):
     """Translate an error code to a message string."""
     try:
-        return space.newtext(_strerror(code))
+        return space.newunicode(_strerror(code))
     except ValueError:
         raise oefmt(space.w_ValueError, "strerror() argument out of range")
 

From pypy.commits at gmail.com  Wed Feb 15 13:14:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 10:14:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: more fixes
Message-ID: <58a49a6b.c9bf1c0a.c58ea.1bd8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90157:c851c9167f8e
Date: 2017-02-15 19:13 +0100
http://bitbucket.org/pypy/pypy/changeset/c851c9167f8e/

Log:	more fixes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1517,6 +1517,9 @@
                 return None
             code = 's*'
         if code == 's*':
+            # NOTE: 's*' is almost not used any more inside CPython 3.5.
+            # Try not to use it pointlessly: it accepts unicodes, which
+            # most API in CPython 3.x no longer do.
             if self.isinstance_w(w_obj, self.w_bytes):
                 return StringBuffer(w_obj.bytes_w(self))
             if self.isinstance_w(w_obj, self.w_unicode):
@@ -1526,6 +1529,9 @@
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or buffer", w_obj)
         elif code == 's#':
+            # NOTE: 's#' is almost not used any more inside CPython 3.5.
+            # Try not to use it pointlessly: it accepts unicodes, which
+            # most API in CPython 3.x no longer do.
             if self.isinstance_w(w_obj, self.w_bytes):
                 return w_obj.bytes_w(self)
             if self.isinstance_w(w_obj, self.w_unicode):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1123,7 +1123,7 @@
         if not space.config.translating:
             fn.add_to_table()
         if gateway.as_classmethod:
-            fn = ClassMethod(space.wrap(fn))
+            fn = ClassMethod(fn)
         #
         from pypy.module.sys.vm import exc_info
         if code._bltin is exc_info:
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -83,7 +83,7 @@
         w_dict = self.w_dict
         space.setitem(w_dict, space.new_interned_str('__name__'), w_name)
         space.setitem(w_dict, space.new_interned_str('__doc__'), w_doc)
-        init_extra_module_attrs(space, space.wrap(self))
+        init_extra_module_attrs(space, self)
 
     def descr__reduce__(self, space):
         w_name = space.finditem(self.w_dict, space.newtext('__name__'))
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -46,7 +46,7 @@
         if not isinstance(w_other, Cell):
             return space.w_NotImplemented
         if self.w_value is None or w_other.w_value is None:
-            return space.wrap(self.w_value == w_other.w_value)
+            return space.newbool(self.w_value == w_other.w_value)
         return space.eq(self.w_value, w_other.w_value)
 
     def descr__reduce__(self, space):
diff --git a/pypy/interpreter/pycompiler.py b/pypy/interpreter/pycompiler.py
--- a/pypy/interpreter/pycompiler.py
+++ b/pypy/interpreter/pycompiler.py
@@ -140,7 +140,7 @@
             validate.validate_ast(self.space, node)
         except validate.ValidationError as e:
             raise OperationError(self.space.w_ValueError,
-                                 self.space.wrap(e.message))
+                                 self.space.newtext(e.message))
 
     def compile_to_ast(self, source, filename, mode, flags):
         info = pyparse.CompileInfo(filename, mode, flags)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -533,7 +533,7 @@
         i = varindex - len(self.pycode.co_cellvars)
         assert i >= 0
         name = self.pycode.co_freevars[i]
-        w_value = space.finditem(self.debugdata.w_locals, space.wrap(name))
+        w_value = space.finditem(self.debugdata.w_locals, space.newtext(name))
         if w_value is None:
             self.LOAD_DEREF(varindex, next_instr)
         else:
@@ -726,7 +726,7 @@
         operror = OperationError(w_type, w_value)
         operror.normalize_exception(space)
         operror.set_cause(space, w_cause)
-        tb = space.getattr(w_value, space.wrap('__traceback__'))
+        tb = space.getattr(w_value, space.newtext('__traceback__'))
         if not space.is_w(tb, space.w_None):
             operror.set_traceback(tb)
         raise operror
@@ -1584,11 +1584,11 @@
                 raise oefmt(space.w_TypeError,
                             "'async for' received an invalid object "
                             "from __aiter__: %T", w_iter)
-            space.warn(space.wrap(
-                u"'%s' implements legacy __aiter__ protocol; "
-                u"__aiter__ should return an asynchronous "
-                u"iterator, not awaitable" %
-                    space.type(w_obj).name.decode('utf-8')),
+            space.warn(space.newtext(
+                "'%s' implements legacy __aiter__ protocol; "
+                "__aiter__ should return an asynchronous "
+                "iterator, not awaitable" %
+                    space.type(w_obj).name),
                 space.w_PendingDeprecationWarning)
         self.pushvalue(w_awaitable)
 
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -50,7 +50,7 @@
         self.next = space.interp_w(PyTraceback, w_next, can_be_None=True)
 
     def descr__dir__(self, space):
-        return space.newlist([space.wrap(n) for n in
+        return space.newlist([space.newtext(n) for n in
             ['tb_frame', 'tb_next', 'tb_lasti', 'tb_lineno']])
 
 
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -53,7 +53,7 @@
     def find(self, w_tofind, w_start=None, w_end=None):
         self.check_valid()
         space = self.space
-        tofind = space.getarg_w('s#', w_tofind)
+        tofind = space.getarg_w('y*', w_tofind)
         if w_start is None:
             start = self.mmap.pos
         else:
@@ -67,7 +67,7 @@
     def rfind(self, w_tofind, w_start=None, w_end=None):
         self.check_valid()
         space = self.space
-        tofind = space.getarg_w('s#', w_tofind)
+        tofind = space.getarg_w('y*', w_tofind)
         if w_start is None:
             start = self.mmap.pos
         else:
@@ -99,7 +99,7 @@
 
     def write(self, w_data):
         self.check_valid()
-        data = self.space.getarg_w('s#', w_data)
+        data = self.space.getarg_w('y*', w_data)
         self.check_writeable()
         try:
             self.mmap.write(data)
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -102,7 +102,7 @@
     return do_unpack(space, format, w_str)
 
 def do_unpack(space, format, w_str):
-    buf = space.getarg_w('s*', w_str)
+    buf = space.getarg_w('y*', w_str)
     return _unpack(space, format, buf)
 
 
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -264,7 +264,8 @@
         assert pack(">?", False) == b'\x00'
         assert pack("@?", True) == b'\x01'
         assert pack("@?", False) == b'\x00'
-        assert self.struct.unpack("?", 'X')[0] is True
+        assert self.struct.unpack("?", b'X')[0] is True
+        raises(TypeError, self.struct.unpack, "?", 'X')
 
     def test_transitiveness(self):
         c = b'a'

From pypy.commits at gmail.com  Wed Feb 15 13:27:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 10:27:27 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix
Message-ID: <58a49d8f.2da9df0a.fcf9b.f343@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90158:d17ec969e353
Date: 2017-02-15 19:17 +0100
http://bitbucket.org/pypy/pypy/changeset/d17ec969e353/

Log:	fix

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -70,7 +70,7 @@
     elif encodingType == PKCS_7_ASN_ENCODING:
         return space.newtext("pkcs_7_asn")
     else:
-        return space.newtext(encodingType)
+        return space.newint(encodingType)
 
 def w_parseKeyUsage(space, pCertCtx, flags):
     with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr:

From pypy.commits at gmail.com  Wed Feb 15 15:28:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 12:28:14 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: windows fix
Message-ID: <58a4b9de.0a0a1c0a.2d832.4985@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90159:ffb3740400e8
Date: 2017-02-15 21:27 +0100
http://bitbucket.org/pypy/pypy/changeset/ffb3740400e8/

Log:	windows fix

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -428,7 +428,7 @@
         if space.isinstance_w(w_path, space.w_unicode):
             path = FileEncoder(space, w_path)
             fullpath = rposix.getfullpathname(path)
-            w_fullpath = space.newtext(fullpath)
+            w_fullpath = space.newunicode(fullpath)
         else:
             path = space.str0_w(w_path)
             fullpath = rposix.getfullpathname(path)

From pypy.commits at gmail.com  Wed Feb 15 15:57:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Feb 2017 12:57:13 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: next fix
Message-ID: <58a4c0a9.53a1df0a.a8b50.1476@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90160:e53e4fe0c683
Date: 2017-02-15 21:56 +0100
http://bitbucket.org/pypy/pypy/changeset/e53e4fe0c683/

Log:	next fix

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -581,7 +581,7 @@
             result_w = [None] * len_result
             for i in range(len_result):
                 res = result[i]
-                w_bytes = space.newtext(res)
+                w_bytes = space.newunicode(res)
                 try:
                     result_w[i] = space.call_method(w_bytes,
                                                     "decode", w_fs_encoding)

From pypy.commits at gmail.com  Wed Feb 15 23:12:59 2017
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Wed, 15 Feb 2017 20:12:59 -0800 (PST)
Subject: [pypy-commit] pypy default: Add missing newline in -vv output
Message-ID: <58a526cb.69acdf0a.29c3f.f1e1@mx.google.com>

Author: William ML Leslie <william.leslie.ttg at gmail.com>
Branch: 
Changeset: r90161:03cbc0865e39
Date: 2017-02-16 15:11 +1100
http://bitbucket.org/pypy/pypy/changeset/03cbc0865e39/

Log:	Add missing newline in -vv output

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -576,7 +576,7 @@
 
             path = space.str0_w(w_pathitem)
             filepart = os.path.join(path, partname)
-            log_pyverbose(space, 2, "# trying %s" % (filepart,))
+            log_pyverbose(space, 2, "# trying %s\n" % (filepart,))
             if os.path.isdir(filepart) and case_ok(filepart):
                 if has_init_module(space, filepart):
                     return FindInfo(PKG_DIRECTORY, filepart, None)

From pypy.commits at gmail.com  Thu Feb 16 10:04:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 07:04:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Untested,
 but it seems we did two typos when copying the CPython logic
Message-ID: <58a5bf66.dea2df0a.e85df.05b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90163:9eb5a1177ff4
Date: 2017-02-16 13:40 +0100
http://bitbucket.org/pypy/pypy/changeset/9eb5a1177ff4/

Log:	Untested, but it seems we did two typos when copying the CPython
	logic

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -576,7 +576,7 @@
             shape = self.getshape()
             strides = self.getstrides()
             if shape[0] == 1 or strides[0] == self.getitemsize():
-                flags |= MEMORYVIEW_C | MEMORYVIEW_SCALAR
+                flags |= MEMORYVIEW_C | MEMORYVIEW_FORTRAN
         else:
             ndim = self.getndim()
             shape = self.getshape()
@@ -587,7 +587,7 @@
                 flags |= MEMORYVIEW_C
             if PyBuffer_isContiguous(None, ndim, shape, strides,
                                       itemsize, 'F'):
-                flags |= MEMORYVIEW_C
+                flags |= MEMORYVIEW_FORTRAN
 
         if self.suboffsets:
             flags |= MEMORYVIEW_PIL

From pypy.commits at gmail.com  Thu Feb 16 10:04:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 07:04:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test and "fix" with ample comments about
 the memoryview() of a ctypes object
Message-ID: <58a5bf68.2d88df0a.c0b1d.0aa6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90164:cb5fc958f45b
Date: 2017-02-16 14:08 +0100
http://bitbucket.org/pypy/pypy/changeset/cb5fc958f45b/

Log:	test and "fix" with ample comments about the memoryview() of a
	ctypes object

diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -17,8 +17,18 @@
     def getformat(self):
         return self.datainstance.fmt
 
-    def getitemsize(self):
-        return self.datainstance.itemsize
+    #XXX we keep the default of 1 for now.  I *think* it does not make
+    # sense to give another answer here without also tweaking the
+    # 'shape' and 'strides'.  At least it makes memoryobject.py think the
+    # buffer is not C-contiguous, which is nonsense (e.g. cast() are
+    # refused).  Now, the memoryview we get from a ctypes object is the
+    # one that would correspond to an array of chars of the same
+    # size---which is wrong, because ctypes gives a more complicated
+    # result on CPython (at least 3.5), but at least it corresponds to
+    # the basics.  (For example, CPython 3.5 gives a zero-dimensional
+    # memoryview if the ctypes type is not an array.)
+    #def getitemsize(self):
+    #    return self.datainstance.itemsize
 
     def getitem(self, index):
         ll_buffer = self.datainstance.ll_buffer
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -8,7 +8,7 @@
 from pypy.conftest import option
 
 class AppTestMemoryView:
-    spaceconfig = dict(usemodules=['array', 'sys'])
+    spaceconfig = dict(usemodules=['array', 'sys', '_rawffi'])
 
     def test_basic(self):
         v = memoryview(b"abc")
@@ -280,6 +280,18 @@
         assert m2.itemsize == m1.itemsize
         assert m2.shape == m1.shape
 
+    def test_cast_ctypes(self):
+        import _rawffi, sys
+        a = _rawffi.Array('i')(1)
+        a[0] = 0x01234567
+        m = memoryview(a).cast('B')
+        if sys.byteorder == 'little':
+            expected = 0x67, 0x45, 0x23, 0x01
+        else:
+            expected = 0x01, 0x23, 0x45, 0x67
+        assert (m[0], m[1], m[2], m[3]) == expected
+        a.free()
+
 class MockBuffer(Buffer):
     def __init__(self, space, w_arr, w_dim, w_fmt, \
                  w_itemsize, w_strides, w_shape):

From pypy.commits at gmail.com  Thu Feb 16 10:04:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 07:04:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: always check if cpython 3.6 fixed a
 cpython 3.5 test that was only
Message-ID: <58a5bf64.09b31c0a.6249c.302f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90162:728c596fc4ed
Date: 2017-02-16 12:59 +0100
http://bitbucket.org/pypy/pypy/changeset/728c596fc4ed/

Log:	always check if cpython 3.6 fixed a cpython 3.5 test that was only
	passing because of randomness in dictionary order

diff --git a/lib-python/3/test/test_weakref.py b/lib-python/3/test/test_weakref.py
--- a/lib-python/3/test/test_weakref.py
+++ b/lib-python/3/test/test_weakref.py
@@ -1381,13 +1381,16 @@
         o = Object(123456)
         with testcontext():
             n = len(dict)
-            dict.popitem()
+            # Since underlaying dict is ordered, first item is popped
+            dict.pop(next(dict.keys()))
             self.assertEqual(len(dict), n - 1)
             dict[o] = o
             self.assertEqual(len(dict), n)
+        # last item in objects is removed from dict in context shutdown
         with testcontext():
             self.assertEqual(len(dict), n - 1)
-            dict.pop(next(dict.keys()))
+            # Then, (o, o) is popped
+            dict.popitem()
             self.assertEqual(len(dict), n - 2)
         with testcontext():
             self.assertEqual(len(dict), n - 3)
@@ -1421,9 +1424,12 @@
                 yield Object(v), v
             finally:
                 it = None           # should commit all removals
-        if not support.check_impl_detail(pypy=True):
-            # XXX: http://bugs.python.org/issue21173
-            self.check_weak_destroy_and_mutate_while_iterating(dict, testcontext)
+                gc.collect()
+        self.check_weak_destroy_and_mutate_while_iterating(dict, testcontext)
+        # Issue #21173: len() fragile when keys are both implicitly and
+        # explicitly removed.
+        dict, objects = self.make_weak_keyed_dict()
+        self.check_weak_del_and_len_while_iterating(dict, testcontext)
 
     def test_weak_values_destroy_while_iterating(self):
         # Issue #7105: iterators shouldn't crash when a key is implicitly removed

From pypy.commits at gmail.com  Thu Feb 16 10:33:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 07:33:00 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: Ah,
 found the proper fix for e53e4fe0c683, which is also a bug on
Message-ID: <58a5c62c.8ba4df0a.24a67.105d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90165:93c00a9b8f71
Date: 2017-02-16 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/93c00a9b8f71/

Log:	Ah, found the proper fix for e53e4fe0c683, which is also a bug on
	Windows on PyPy2: if listdir(unicode) returns RPython unicodes, then
	they must not be decode()d again!

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -576,20 +576,28 @@
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
             result = rposix.listdir(dirname)
+            # NOTE: 'result' can be either a list of str or a list of
+            # unicodes, depending on the platform
             w_fs_encoding = getfilesystemencoding(space)
             len_result = len(result)
             result_w = [None] * len_result
             for i in range(len_result):
                 res = result[i]
-                w_bytes = space.newunicode(res)
-                try:
-                    result_w[i] = space.call_method(w_bytes,
-                                                    "decode", w_fs_encoding)
-                except OperationError as e:
-                    # fall back to the original byte string
-                    if e.async(space):
-                        raise
-                    result_w[i] = w_bytes
+                if isinstance(res, str):
+                    w_bytes = space.newtext(res)
+                    try:
+                        w_res = space.call_method(w_bytes,
+                                                  "decode", w_fs_encoding)
+                    except OperationError as e:
+                        # fall back to the original byte string
+                        if e.async(space):
+                            raise
+                        w_res = w_bytes
+                elif isinstance(res, unicode):
+                    w_res = space.newunicode(res)
+                else:
+                    assert False
+                result_w[i] = w_res
             return space.newlist(result_w)
         else:
             dirname = space.str0_w(w_dirname)

From pypy.commits at gmail.com  Thu Feb 16 10:42:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 07:42:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: hg merge space-newtext
Message-ID: <58a5c87d.066b1c0a.ebb75.4148@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90166:010533436bdc
Date: 2017-02-16 16:42 +0100
http://bitbucket.org/pypy/pypy/changeset/010533436bdc/

Log:	hg merge space-newtext

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -695,7 +695,7 @@
         if space.isinstance_w(w_path, space.w_unicode):
             path = FileEncoder(space, w_path)
             fullpath = rposix.getfullpathname(path)
-            w_fullpath = space.newtext(fullpath)
+            w_fullpath = space.newunicode(fullpath)
         else:
             path = space.str0_w(w_path)
             fullpath = rposix.getfullpathname(path)

From pypy.commits at gmail.com  Thu Feb 16 12:27:45 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 16 Feb 2017 09:27:45 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-2: fix bugs in revision
 e1832b3ddde9
Message-ID: <58a5e111.28a8df0a.9df97.32e8@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-2
Changeset: r90167:a8f437494e4d
Date: 2017-02-16 17:01 +0000
http://bitbucket.org/pypy/pypy/changeset/a8f437494e4d/

Log:	fix bugs in revision e1832b3ddde9

	(branch to debug problems with optinfo-into-bridges)

diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1266,9 +1266,6 @@
         assert box.type == kind
         return box
 
-    def next_box_of_type(self, kind):
-        return self.decode_box(self.resumecodereader.next_item(), kind)
-
     def load_box_from_cpu(self, num, kind):
         if num < 0:
             num += len(self.liveboxes)
@@ -1297,7 +1294,7 @@
             kind = FLOAT
         else:
             raise AssertionError(kind)
-        return self.decode_box(tagged, kind)
+        return self.decode_box(self.resumecodereader.next_item(), kind)
     next_box_of_type._annspecialcase_ = 'specialize:arg(1)'
 
     def write_an_int(self, index, box):
@@ -1408,7 +1405,6 @@
         # just reset the token, we'll force it later
         vinfo.reset_token_gcref(virtualizable)
         vinfo.write_from_resume_data_partial(virtualizable, self)
-        return index
 
     def load_next_value_of_type(self, TYPE):
         from rpython.jit.metainterp.warmstate import specialize_value
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -80,7 +80,7 @@
     return l
 
 class Writer(object):
-    def __init__(self, size):
+    def __init__(self, size=0):
         self.current = objectmodel.newlist_hint(3 * size)
         self.grow(size)
         self.items = 0
@@ -111,6 +111,13 @@
         self.current = l + self.current[1:]
 
 
+def create_numbering(l):
+    w = Writer()
+    for item in l:
+        w.append_int(item)
+    return w.create_numbering()
+
+
 class Reader(object):
     def __init__(self, code):
         self.code = code

From pypy.commits at gmail.com  Thu Feb 16 13:16:51 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 16 Feb 2017 10:16:51 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: add Rober Zaremba to the people
 list for this years sprint
Message-ID: <58a5ec93.84301c0a.e740e.63a0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5781:e9c5b527c995
Date: 2017-02-16 19:16 +0100
http://bitbucket.org/pypy/extradoc/changeset/e9c5b527c995/

Log:	add Rober Zaremba to the people list for this years sprint

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -13,6 +13,7 @@
 Richard Plangger     26.02/04.03    Ermina
 Remi Meier           (?)27.02/04.03 Ermina
 Manuel Jacob         25.02/04.03    Ermina
+Robert Zaremba       25.02/01.03    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Thu Feb 16 14:56:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 11:56:46 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: Close branch for merge
Message-ID: <58a603fe.e30f190a.19f1e.8aae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: space-newtext
Changeset: r90168:cff8c0d9d9da
Date: 2017-02-16 20:47 +0100
http://bitbucket.org/pypy/pypy/changeset/cff8c0d9d9da/

Log:	Close branch for merge


From pypy.commits at gmail.com  Thu Feb 16 14:56:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 11:56:52 -0800 (PST)
Subject: [pypy-commit] pypy default: hg merge space-newtext
Message-ID: <58a60404.15502e0a.10271.8e10@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90169:9aa4348f6606
Date: 2017-02-16 20:50 +0100
http://bitbucket.org/pypy/pypy/changeset/9aa4348f6606/

Log:	hg merge space-newtext

	A branch (by cfbolz) that tries to remove all usages of space.wrap()
	in translation, as well as space.str_w() and a few other methods
	that are ambiguous across py2-py3 bytes-str-unicodes. May not be
	completely finished, but let's merge anyway and finish it
	incrementally.

diff too long, truncating to 2000 out of 26698 lines

diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -188,6 +188,7 @@
 
 .. py:function:: wrap(x)
 
+   **Deprecated! Eventually this method should disappear.**
    Returns a wrapped object that is a reference to the interpreter-level object
    :py:obj:`x`. This can be used either on simple immutable objects (integers,
    strings, etc) to create a new wrapped object, or on instances of :py:class:`W_Root`
@@ -196,6 +197,35 @@
    be directly exposed to application-level code in this way - functions, frames,
    code objects, etc.
 
+.. py:function:: newint(i)
+
+   Creates a wrapped object holding an integral value. `newint` creates an object
+   of type `W_IntObject`.
+
+.. py:function:: newlong(l)
+
+   Creates a wrapped object holding an integral value. The main difference to newint
+   is the type of the argument (which is rpython.rlib.rbigint.rbigint). On PyPy3 this
+   method will return an :py:class:`int` (PyPy2 it returns a :py:class:`long`).
+
+.. py:function:: newbytes(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`bytes` (both on PyPy2 and PyPy3).
+
+.. py:function:: newtext(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`str`.  On PyPy3 this will return a wrapped unicode
+   object. The object will hold a utf-8-nosg decoded value of `t`.
+   The "utf-8-nosg" codec used here is slightly different from the
+   "utf-8" implemented in Python 2 or Python 3: it is defined as utf-8
+   without any special handling of surrogate characters.  They are
+   encoded using the same three-bytes sequence that encodes any char in
+   the range from ``'\u0800'`` to ``'\uffff'``.
+
+   PyPy2 will return a bytestring object. No encoding/decoding steps will be applied.
+
 .. py:function:: newbool(b)
 
    Creates a wrapped :py:class:`bool` object from an :ref:`interpreter-level <interpreter-level>`
@@ -217,15 +247,18 @@
 
    Creates a new slice object.
 
-.. py:function:: newstring(asciilist)
+.. py:function:: newunicode(ustr)
 
-   Creates a string from a list of wrapped integers. Note that this may not be
-   a very useful method; usually you can just write ``space.wrap("mystring")``.
+   Creates a Unicode string from an rpython unicode string.
+   This method may disappear soon and be replaced by :py:function:`newutf8()`.
 
-.. py:function:: newunicode(codelist)
+.. py:function:: newutf8(bytestr)
 
-   Creates a Unicode string from a list of integers (code points).
+   Creates a Unicode string from an rpython byte string, decoded as
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
 
+Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
+`pypy/objspace/std/objspace.py`.
 
 Conversions from Application Level to Interpreter Level
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -258,11 +291,28 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: text_w(w_x)
+
+   Takes an application level :py:class:`str` and converts it to a rpython byte string.
+   PyPy3 this method will return an utf-8-nosg encoded result.
+
+.. py:function:: bytes_w(w_x)
+
+   Takes an application level :py:class:`bytes` (PyPy2 this equals `str`) and returns a rpython
+   byte string.
+
 .. py:function:: str_w(w_x)
 
+   **Deprecated. use text_w or bytes_w instead**
    If :py:obj:`w_x` is an application-level string, return an interpreter-level string.
    Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: unicode_w(w_x)
+
+   Takes an application level :py:class:`unicode` and return an
+   interpreter-level unicode string.  This method may disappear soon and
+   be replaced by :py:function:`text_w()`.
+
 .. py:function:: float_w(w_x)
 
    If :py:obj:`w_x` is an application-level float, integer or long, return an
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -31,9 +31,9 @@
 
 def create_entry_point(space, w_dict):
     if w_dict is not None: # for tests
-        w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
-        w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
-        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
+        w_entry_point = space.getitem(w_dict, space.newtext('entry_point'))
+        w_run_toplevel = space.getitem(w_dict, space.newtext('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.newtext('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
         hashfunc = space.config.objspace.hash
     else:
@@ -65,8 +65,8 @@
         try:
             try:
                 space.startup()
-                w_executable = space.wrap(argv[0])
-                w_argv = space.newlist([space.wrap(s) for s in argv[1:]])
+                w_executable = space.newtext(argv[0])
+                w_argv = space.newlist([space.newtext(s) for s in argv[1:]])
                 w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
                 exitcode = space.int_w(w_exitcode)
                 # try to pull it all in
@@ -76,7 +76,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
             try:
@@ -84,7 +84,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
 
@@ -123,7 +123,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap(home), w_initstdio],
+            space.appexec([w_path, space.newtext(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path
@@ -141,7 +141,7 @@
             if verbose:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
             if must_leave:
@@ -180,11 +180,11 @@
     def _pypy_execute_source(source, c_argument):
         try:
             w_globals = space.newdict(module=True)
-            space.setitem(w_globals, space.wrap('__builtins__'),
+            space.setitem(w_globals, space.newtext('__builtins__'),
                           space.builtin_modules['__builtin__'])
-            space.setitem(w_globals, space.wrap('c_argument'),
-                          space.wrap(c_argument))
-            space.appexec([space.wrap(source), w_globals], """(src, glob):
+            space.setitem(w_globals, space.newtext('c_argument'),
+                          space.newint(c_argument))
+            space.appexec([space.newtext(source), w_globals], """(src, glob):
                 import sys
                 stmt = compile(src, 'c callback', 'exec')
                 if not hasattr(sys, '_pypy_execute_source'):
@@ -195,7 +195,7 @@
         except OperationError as e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space))
-            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
 
@@ -323,7 +323,7 @@
         # obscure hack to stuff the translation options into the translated PyPy
         import pypy.module.sys
         options = make_dict(config)
-        wrapstr = 'space.wrap(%r)' % (options)
+        wrapstr = 'space.wrap(%r)' % (options) # import time
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -363,7 +363,7 @@
     i = 0
     for w_key in keys_w:
         try:
-            key = space.str_w(w_key)
+            key = space.text_w(w_key)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError, "keywords must be strings")
@@ -547,11 +547,11 @@
                         except IndexError:
                             name = '?'
                         else:
-                            w_enc = space.wrap(space.sys.defaultencoding)
-                            w_err = space.wrap("replace")
+                            w_enc = space.newtext(space.sys.defaultencoding)
+                            w_err = space.newtext("replace")
                             w_name = space.call_method(w_name, "encode", w_enc,
                                                        w_err)
-                            name = space.str_w(w_name)
+                            name = space.text_w(w_name)
                     break
         self.kwd_name = name
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -266,8 +266,8 @@
             else:
                 w_key = space.newtuple([obj, space.w_float])
         elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
+            w_real = space.getattr(obj, space.newtext("real"))
+            w_imag = space.getattr(obj, space.newtext("imag"))
             real = space.float_w(w_real)
             imag = space.float_w(w_imag)
             real_negzero = (real == 0.0 and
@@ -366,7 +366,7 @@
         space = self.space
         consts_w = [space.w_None] * space.len_w(w_consts)
         w_iter = space.iter(w_consts)
-        first = space.wrap(0)
+        first = space.newint(0)
         while True:
             try:
                 w_key = space.next(w_iter)
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -13,7 +13,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -48,10 +48,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
-
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+        assert fields == []
+
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -67,14 +67,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -93,12 +93,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -114,7 +114,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -143,16 +143,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)
+            space.newtext(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
@@ -196,7 +196,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -230,7 +230,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -258,7 +258,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expression)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -293,7 +293,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -390,26 +390,26 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FunctionDef)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -463,30 +463,30 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ClassDef)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.bases is None:
             bases_w = []
         else:
             bases_w = [node.to_object(space) for node in self.bases] # expr
         w_bases = space.newlist(bases_w)
-        space.setattr(w_node, space.wrap('bases'), w_bases)
+        space.setattr(w_node, space.newtext('bases'), w_bases)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -530,11 +530,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -572,11 +572,11 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -617,13 +617,13 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -663,15 +663,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AugAssign)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -719,19 +719,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Print)
         w_dest = self.dest.to_object(space) if self.dest is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('dest'), w_dest)
+        space.setattr(w_node, space.newtext('dest'), w_dest)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_nl = space.wrap(self.nl)  # bool
-        space.setattr(w_node, space.wrap('nl'), w_nl)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_nl = space.newbool(self.nl)  # bool
+        space.setattr(w_node, space.newtext('nl'), w_nl)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -780,25 +780,25 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_For)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -850,23 +850,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_While)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -914,23 +914,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_If)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -977,19 +977,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_With)
         w_context_expr = self.context_expr.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
+        space.setattr(w_node, space.newtext('context_expr'), w_context_expr)
         w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
+        space.setattr(w_node, space.newtext('optional_vars'), w_optional_vars)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1035,15 +1035,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
+        space.setattr(w_node, space.newtext('type'), w_type)
         w_inst = self.inst.to_object(space) if self.inst is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('inst'), w_inst)
+        space.setattr(w_node, space.newtext('inst'), w_inst)
         w_tback = self.tback.to_object(space) if self.tback is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('tback'), w_tback)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('tback'), w_tback)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1093,23 +1093,23 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.handlers is None:
             handlers_w = []
         else:
             handlers_w = [node.to_object(space) for node in self.handlers] # excepthandler
         w_handlers = space.newlist(handlers_w)
-        space.setattr(w_node, space.wrap('handlers'), w_handlers)
+        space.setattr(w_node, space.newtext('handlers'), w_handlers)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1158,17 +1158,17 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.finalbody is None:
             finalbody_w = []
         else:
             finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt
         w_finalbody = space.newlist(finalbody_w)
-        space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('finalbody'), w_finalbody)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1207,13 +1207,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Assert)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_msg = self.msg.to_object(space) if self.msg is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('msg'), w_msg)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('msg'), w_msg)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1255,11 +1255,11 @@
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1295,20 +1295,20 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = space.wrap(self.module)  # identifier
-        space.setattr(w_node, space.wrap('module'), w_module)
+        w_module = space.newtext_or_none(self.module)  # identifier
+        space.setattr(w_node, space.newtext('module'), w_module)
         if self.names is None:
             names_w = []
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_level = space.wrap(self.level)  # int
-        space.setattr(w_node, space.wrap('level'), w_level)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_level = space.newint(self.level)  # int
+        space.setattr(w_node, space.newtext('level'), w_level)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1351,15 +1351,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Exec)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_globals = self.globals.to_object(space) if self.globals is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('globals'), w_globals)
+        space.setattr(w_node, space.newtext('globals'), w_globals)
         w_locals = self.locals.to_object(space) if self.locals is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('locals'), w_locals)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('locals'), w_locals)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1398,13 +1398,13 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [space.wrap(node) for node in self.names] # identifier
+            names_w = [space.newtext(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1437,11 +1437,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1472,10 +1472,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Pass)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1502,10 +1502,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Break)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1532,10 +1532,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Continue)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1628,17 +1628,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BoolOp)
         w_op = boolop_to_class[self.op - 1]().to_object(space)  # boolop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1678,15 +1678,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BinOp)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_right = self.right.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('right'), w_right)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('right'), w_right)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1729,13 +1729,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_UnaryOp)
         w_op = unaryop_to_class[self.op - 1]().to_object(space)  # unaryop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_operand = self.operand.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('operand'), w_operand)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('operand'), w_operand)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1775,13 +1775,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Lambda)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1823,15 +1823,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_IfExp)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_orelse = self.orelse.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1883,17 +1883,17 @@
         else:
             keys_w = [node.to_object(space) for node in self.keys] # expr
         w_keys = space.newlist(keys_w)
-        space.setattr(w_node, space.wrap('keys'), w_keys)
+        space.setattr(w_node, space.newtext('keys'), w_keys)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1935,11 +1935,11 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1976,17 +1976,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ListComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2027,17 +2027,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_SetComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2080,19 +2080,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_DictComp)
         w_key = self.key.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('key'), w_key)
+        space.setattr(w_node, space.newtext('key'), w_key)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2137,17 +2137,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_GeneratorExp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2185,11 +2185,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2226,23 +2226,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Compare)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         if self.ops is None:
             ops_w = []
         else:
             ops_w = [cmpop_to_class[node - 1]().to_object(space) for node in self.ops] # cmpop
         w_ops = space.newlist(ops_w)
-        space.setattr(w_node, space.wrap('ops'), w_ops)
+        space.setattr(w_node, space.newtext('ops'), w_ops)
         if self.comparators is None:
             comparators_w = []
         else:
             comparators_w = [node.to_object(space) for node in self.comparators] # expr
         w_comparators = space.newlist(comparators_w)
-        space.setattr(w_node, space.wrap('comparators'), w_comparators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('comparators'), w_comparators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2296,27 +2296,27 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Call)
         w_func = self.func.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('func'), w_func)
+        space.setattr(w_node, space.newtext('func'), w_func)
         if self.args is None:
             args_w = []
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.keywords is None:
             keywords_w = []
         else:
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
-        space.setattr(w_node, space.wrap('keywords'), w_keywords)
+        space.setattr(w_node, space.newtext('keywords'), w_keywords)
         w_starargs = self.starargs.to_object(space) if self.starargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('starargs'), w_starargs)
+        space.setattr(w_node, space.newtext('starargs'), w_starargs)
         w_kwargs = self.kwargs.to_object(space) if self.kwargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('kwargs'), w_kwargs)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2360,11 +2360,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Repr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2397,11 +2397,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Num)
         w_n = self.n  # object
-        space.setattr(w_node, space.wrap('n'), w_n)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('n'), w_n)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2434,11 +2434,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Str)
         w_s = self.s  # string
-        space.setattr(w_node, space.wrap('s'), w_s)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('s'), w_s)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2474,15 +2474,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Attribute)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_attr = space.wrap(self.attr)  # identifier
-        space.setattr(w_node, space.wrap('attr'), w_attr)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_attr = space.newtext(self.attr)  # identifier
+        space.setattr(w_node, space.newtext('attr'), w_attr)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2527,15 +2527,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Subscript)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         w_slice = self.slice.to_object(space)  # slice
-        space.setattr(w_node, space.wrap('slice'), w_slice)
+        space.setattr(w_node, space.newtext('slice'), w_slice)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2576,14 +2576,14 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Name)
-        w_id = space.wrap(self.id)  # identifier
-        space.setattr(w_node, space.wrap('id'), w_id)
+        w_id = space.newtext(self.id)  # identifier
+        space.setattr(w_node, space.newtext('id'), w_id)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2628,13 +2628,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2678,13 +2678,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2720,11 +2720,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Const)
         w_value = self.value  # object
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2866,11 +2866,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
         w_lower = self.lower.to_object(space) if self.lower is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('lower'), w_lower)
+        space.setattr(w_node, space.newtext('lower'), w_lower)
         w_upper = self.upper.to_object(space) if self.upper is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('upper'), w_upper)
+        space.setattr(w_node, space.newtext('upper'), w_upper)
         w_step = self.step.to_object(space) if self.step is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('step'), w_step)
+        space.setattr(w_node, space.newtext('step'), w_step)
         return w_node
 
     @staticmethod
@@ -2907,7 +2907,7 @@
         else:
             dims_w = [node.to_object(space) for node in self.dims] # slice
         w_dims = space.newlist(dims_w)
-        space.setattr(w_node, space.wrap('dims'), w_dims)
+        space.setattr(w_node, space.newtext('dims'), w_dims)
         return w_node
 
     @staticmethod
@@ -2935,7 +2935,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Index)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
@@ -3266,15 +3266,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_comprehension)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.ifs is None:
             ifs_w = []
         else:
             ifs_w = [node.to_object(space) for node in self.ifs] # expr
         w_ifs = space.newlist(ifs_w)
-        space.setattr(w_node, space.wrap('ifs'), w_ifs)
+        space.setattr(w_node, space.newtext('ifs'), w_ifs)
         return w_node
 
     @staticmethod
@@ -3334,19 +3334,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
+        space.setattr(w_node, space.newtext('type'), w_type)
         w_name = self.name.to_object(space) if self.name is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('name'), w_name)
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3394,17 +3394,17 @@
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
-        w_vararg = space.wrap(self.vararg)  # identifier
-        space.setattr(w_node, space.wrap('vararg'), w_vararg)
-        w_kwarg = space.wrap(self.kwarg)  # identifier
-        space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
+        space.setattr(w_node, space.newtext('args'), w_args)
+        w_vararg = space.newtext_or_none(self.vararg)  # identifier
+        space.setattr(w_node, space.newtext('vararg'), w_vararg)
+        w_kwarg = space.newtext_or_none(self.kwarg)  # identifier
+        space.setattr(w_node, space.newtext('kwarg'), w_kwarg)
         if self.defaults is None:
             defaults_w = []
         else:
             defaults_w = [node.to_object(space) for node in self.defaults] # expr
         w_defaults = space.newlist(defaults_w)
-        space.setattr(w_node, space.wrap('defaults'), w_defaults)
+        space.setattr(w_node, space.newtext('defaults'), w_defaults)
         return w_node
 
     @staticmethod
@@ -3438,10 +3438,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.wrap(self.arg)  # identifier
-        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_arg = space.newtext(self.arg)  # identifier
+        space.setattr(w_node, space.newtext('arg'), w_arg)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
@@ -3472,10 +3472,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_alias)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
-        w_asname = space.wrap(self.asname)  # identifier
-        space.setattr(w_node, space.wrap('asname'), w_asname)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
+        w_asname = space.newtext_or_none(self.asname)  # identifier
+        space.setattr(w_node, space.newtext('asname'), w_asname)
         return w_node
 
     @staticmethod
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1070,8 +1070,8 @@
                 raw = "0" + raw
         if negative:
             raw = "-" + raw
-        w_num_str = self.space.wrap(raw)
-        w_base = self.space.wrap(base)
+        w_num_str = self.space.newtext(raw)
+        w_base = self.space.newint(base)
         if raw[-1] in "lL":
             tp = self.space.w_long
             return self.space.call_function(tp, w_num_str, w_base)
@@ -1110,7 +1110,7 @@
             # This implements implicit string concatenation.
             if len(sub_strings_w) > 1:
                 w_sub_strings = space.newlist(sub_strings_w)
-                w_join = space.getattr(space.wrap(""), space.wrap("join"))
+                w_join = space.getattr(space.newtext(""), space.newtext("join"))
                 final_string = space.call_function(w_join, w_sub_strings)
             else:
                 final_string = sub_strings_w[0]
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -322,7 +322,7 @@
     def visit_ClassDef(self, cls):
         self.update_position(cls.lineno, True)
         self.visit_sequence(cls.decorator_list)
-        self.load_const(self.space.wrap(cls.name))
+        self.load_const(self.space.newtext(cls.name))
         self.visit_sequence(cls.bases)
         bases_count = len(cls.bases) if cls.bases is not None else 0
         self.emit_op_arg(ops.BUILD_TUPLE, bases_count)
@@ -611,7 +611,7 @@
                 level = 0
             else:
                 level = -1
-            self.load_const(self.space.wrap(level))
+            self.load_const(self.space.newint(level))
             self.load_const(self.space.w_None)
             self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name)
             # If there's no asname then we store the root module.  If there is
@@ -654,12 +654,12 @@
             level = -1
         else:
             level = imp.level
-        self.load_const(space.wrap(level))
+        self.load_const(space.newint(level))
         names_w = [None]*len(imp.names)
         for i in range(len(imp.names)):
             alias = imp.names[i]
             assert isinstance(alias, ast.alias)
-            names_w[i] = space.wrap(alias.name)
+            names_w[i] = space.newtext(alias.name)
         self.load_const(space.newtuple(names_w))
         if imp.module:
             mod_name = imp.module
@@ -944,7 +944,7 @@
         self.name_op(name.id, name.ctx)
 
     def visit_keyword(self, keyword):
-        self.load_const(self.space.wrap(keyword.arg))
+        self.load_const(self.space.newtext(keyword.arg))
         keyword.value.walkabout(self)
 
     def visit_Call(self, call):
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -19,10 +19,10 @@
 
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
-    w_msg = space.wrap(msg)
-    w_filename = space.wrap(fn)
-    w_lineno = space.wrap(lineno)
-    w_offset = space.wrap(offset)
+    w_msg = space.newtext(msg)
+    w_filename = space.newtext(fn)
+    w_lineno = space.newint(lineno)
+    w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
 
 
@@ -110,9 +110,9 @@
 
 def intern_if_common_string(space, w_const):
     # only intern identifier-like strings
-    if not space.is_w(space.type(w_const), space.w_str):
+    if not space.is_w(space.type(w_const), space.w_text):
         return w_const
-    for c in space.str_w(w_const):
+    for c in space.text_w(w_const):
         if not (c.isalnum() or c == '_'):
             return w_const
     return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -120,7 +120,7 @@
     return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
-    return space.wrap(not space.is_true(operand))
+    return space.newbool(not space.is_true(operand))
 
 
 binary_folders = {
@@ -216,7 +216,7 @@
                         break
                 else:
                     raise AssertionError("unknown unary operation")
-                w_minint = self.space.wrap(-sys.maxint - 1)
+                w_minint = self.space.newint(-sys.maxint - 1)
                 # This makes sure the result is an integer.
                 if self.space.eq_w(w_minint, w_const):
                     w_const = w_minint
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -127,8 +127,14 @@
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
         elif field.type in ("object", "singleton", "string", "bytes"):
             return value
-        elif field.type in ("identifier", "int", "bool"):
-            return "space.wrap(%s)" % (value,)
+        elif field.type == "bool":
+            return "space.newbool(%s)" % (value,)
+        elif field.type == "int":
+            return "space.newint(%s)" % (value,)
+        elif field.type == "identifier":
+            if field.opt:
+                return "space.newtext_or_none(%s)" % (value,)
+            return "space.newtext(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             if field.opt:
@@ -194,7 +200,7 @@
             wrapping_code = self.get_field_converter(field)
             for line in wrapping_code:
                 self.emit(line, 2)
-            self.emit("space.setattr(w_node, space.wrap(%r), w_%s)" % (
+            self.emit("space.setattr(w_node, space.newtext(%r), w_%s)" % (
                     str(field.name), field.name), 2)
         self.emit("return w_node", 2)
         self.emit("")
@@ -402,7 +408,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -437,10 +443,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
+        assert fields == []
 
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -456,14 +462,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -482,12 +488,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -503,7 +509,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -532,16 +538,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)
+            space.newtext(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -5,7 +5,7 @@
 from rpython.rlib import jit, types
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.objectmodel import (we_are_translated, newlist_hint,
-     compute_unique_id, specialize)
+     compute_unique_id, specialize, not_rpython)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
@@ -51,7 +51,7 @@
         w_dict = self.getdict(space)
         if w_dict is not None:
             try:
-                space.delitem(w_dict, space.wrap(attr))
+                space.delitem(w_dict, space.newtext(attr))
                 return True
             except OperationError as ex:
                 if not ex.match(space, space.w_KeyError):
@@ -77,7 +77,7 @@
 
     def getname(self, space):
         try:
-            return space.str_w(space.getattr(self, space.wrap('__name__')))
+            return space.text_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return '?'
@@ -86,8 +86,8 @@
     def getaddrstring(self, space):
         # slowish
         w_id = space.id(self)
-        w_4 = space.wrap(4)
-        w_0x0F = space.wrap(0x0F)
+        w_4 = space.newint(4)
+        w_0x0F = space.newint(0x0F)
         i = 2 * HUGEVAL_BYTES
         addrstring = [' '] * i
         while True:
@@ -104,8 +104,8 @@
 
     def getrepr(self, space, info, moreinfo=''):
         addrstring = self.getaddrstring(space)
-        return space.wrap("<%s at 0x%s%s>" % (info, addrstring,
-                                              moreinfo))
+        return space.newtext("<%s at 0x%s%s>" % (info, addrstring,
+                                                 moreinfo))
 
     def getslotvalue(self, index):
         raise NotImplementedError
@@ -321,11 +321,14 @@
         raise oefmt(space.w_TypeError,
                     "ord() expected string of length 1, but %T found", self)
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
+        """ Return a version of the object bound to a specific object space
+        instance. This is used for objects (like e.g. TypeDefs) that are
+        constructed before there is an object space instance. """
         return self
 
+    @not_rpython
     def unwrap(self, space):
-        """NOT_RPYTHON"""
         # _____ this code is here to support testing only _____
         return self
 
@@ -408,8 +411,9 @@
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
+    @not_rpython
     def __init__(self, config=None):
-        "NOT_RPYTHON: Basic initialization of objects."
+        "Basic initialization of objects."
         self.fromcache = InternalSpaceCache(self).getorbuild
         self.threadlocals = ThreadLocals()
         # set recursion limit
@@ -487,8 +491,9 @@
         except AttributeError:
             return self.__class__.__name__
 
+    @not_rpython
     def setbuiltinmodule(self, importname):
-        """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules"""
+        """load a lazy pypy/module and put it into sys.modules"""
         if '.' in importname:
             fullname = importname
             importname = fullname.rsplit('.', 1)[1]
@@ -502,13 +507,13 @@
         else:
             name = importname
 
-        mod = Module(self, self.wrap(name))
+        mod = Module(self, self.newtext(name))
         mod.install()
 
         return name
 
     def getbuiltinmodule(self, name, force_init=False, reuse=True):
-        w_name = self.wrap(name)
+        w_name = self.newtext(name)
         w_modules = self.sys.get('modules')
         if not force_init:
             assert reuse
@@ -534,7 +539,7 @@
             if not reuse and w_mod.startup_called:
                 # create a copy of the module.  (see issue1514) eventlet
                 # patcher relies on this behaviour.
-                w_mod2 = self.wrap(Module(self, w_name))
+                w_mod2 = Module(self, w_name)
                 self.setitem(w_modules, w_name, w_mod2)
                 w_mod.getdict(self)  # unlazy w_initialdict
                 self.call_method(w_mod2.getdict(self), 'update',
@@ -546,8 +551,8 @@
             self.setitem(w_modules, w_name, w_mod)
         return w_mod
 
+    @not_rpython
     def get_builtinmodule_to_install(self):
-        """NOT_RPYTHON"""
         try:
             return self._builtinmodule_list
         except AttributeError:
@@ -586,30 +591,31 @@
         'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport',
     ], None)
 
+    @not_rpython
     def make_builtins(self):
-        "NOT_RPYTHON: only for initializing the space."
+        "only for initializing the space."
 
         from pypy.module.exceptions import Module
-        w_name = self.wrap('exceptions')
+        w_name = self.newtext('exceptions')
         self.exceptions_module = Module(self, w_name)
         self.exceptions_module.install()
 
         from pypy.module.sys import Module
-        w_name = self.wrap('sys')
+        w_name = self.newtext('sys')
         self.sys = Module(self, w_name)
         self.sys.install()
 
         from pypy.module.imp import Module
-        w_name = self.wrap('imp')
+        w_name = self.newtext('imp')
         mod = Module(self, w_name)
         mod.install()
 
         from pypy.module.__builtin__ import Module
-        w_name = self.wrap('__builtin__')
+        w_name = self.newtext('__builtin__')
         self.builtin = Module(self, w_name)
         w_builtin = self.wrap(self.builtin)
         w_builtin.install()
-        self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)
+        self.setitem(self.builtin.w_dict, self.newtext('__builtins__'), w_builtin)
 
         bootstrap_modules = set(('sys', 'imp', '__builtin__', 'exceptions'))
         installed_builtin_modules = list(bootstrap_modules)
@@ -620,7 +626,7 @@
         types_w = (self.get_builtin_types().items() +
                    exception_types_w.items())
         for name, w_type in types_w:
-            self.setitem(self.builtin.w_dict, self.wrap(name), w_type)
+            self.setitem(self.builtin.w_dict, self.newtext(name), w_type)
 
         # install mixed modules
         for mixedname in self.get_builtinmodule_to_install():
@@ -632,7 +638,7 @@
             [self.wrap(fn) for fn in installed_builtin_modules])
 
         # force this value into the dict without unlazyfying everything
-        self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'),
+        self.setitem(self.sys.w_dict, self.newtext('builtin_module_names'),
                      w_builtin_module_names)
 
     def get_builtin_types(self):
@@ -640,13 +646,13 @@
         objects."""
         raise NotImplementedError
 
+    @not_rpython
     def export_builtin_exceptions(self):
-        """NOT_RPYTHON"""
         w_dic = self.exceptions_module.getdict(self)
         w_keys = self.call_method(w_dic, "keys")
         exc_types_w = {}
         for w_name in self.unpackiterable(w_keys):
-            name = self.str_w(w_name)
+            name = self.text_w(w_name)
             if not name.startswith('__'):
                 excname = name

From pypy.commits at gmail.com  Thu Feb 16 14:56:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 11:56:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-newtext
Message-ID: <58a6040a.0e152e0a.20484.8ae9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90171:9ef7c5f49f7e
Date: 2017-02-16 20:51 +0100
http://bitbucket.org/pypy/pypy/changeset/9ef7c5f49f7e/

Log:	hg merge py3.5-newtext

	See comments in 9aa4348f6606.

diff too long, truncating to 2000 out of 25082 lines

diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -188,6 +188,7 @@
 
 .. py:function:: wrap(x)
 
+   **Deprecated! Eventually this method should disappear.**
    Returns a wrapped object that is a reference to the interpreter-level object
    :py:obj:`x`. This can be used either on simple immutable objects (integers,
    strings, etc) to create a new wrapped object, or on instances of :py:class:`W_Root`
@@ -196,6 +197,35 @@
    be directly exposed to application-level code in this way - functions, frames,
    code objects, etc.
 
+.. py:function:: newint(i)
+
+   Creates a wrapped object holding an integral value. `newint` creates an object
+   of type `W_IntObject`.
+
+.. py:function:: newlong(l)
+
+   Creates a wrapped object holding an integral value. The main difference to newint
+   is the type of the argument (which is rpython.rlib.rbigint.rbigint). On PyPy3 this
+   method will return an :py:class:`int` (PyPy2 it returns a :py:class:`long`).
+
+.. py:function:: newbytes(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`bytes` (both on PyPy2 and PyPy3).
+
+.. py:function:: newtext(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`str`.  On PyPy3 this will return a wrapped unicode
+   object. The object will hold a utf-8-nosg decoded value of `t`.
+   The "utf-8-nosg" codec used here is slightly different from the
+   "utf-8" implemented in Python 2 or Python 3: it is defined as utf-8
+   without any special handling of surrogate characters.  They are
+   encoded using the same three-bytes sequence that encodes any char in
+   the range from ``'\u0800'`` to ``'\uffff'``.
+
+   PyPy2 will return a bytestring object. No encoding/decoding steps will be applied.
+
 .. py:function:: newbool(b)
 
    Creates a wrapped :py:class:`bool` object from an :ref:`interpreter-level <interpreter-level>`
@@ -217,15 +247,18 @@
 
    Creates a new slice object.
 
-.. py:function:: newstring(asciilist)
+.. py:function:: newunicode(ustr)
 
-   Creates a string from a list of wrapped integers. Note that this may not be
-   a very useful method; usually you can just write ``space.wrap("mystring")``.
+   Creates a Unicode string from an rpython unicode string.
+   This method may disappear soon and be replaced by :py:function:`newutf8()`.
 
-.. py:function:: newunicode(codelist)
+.. py:function:: newutf8(bytestr)
 
-   Creates a Unicode string from a list of integers (code points).
+   Creates a Unicode string from an rpython byte string, decoded as
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
 
+Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
+`pypy/objspace/std/objspace.py`.
 
 Conversions from Application Level to Interpreter Level
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -258,11 +291,28 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: text_w(w_x)
+
+   Takes an application level :py:class:`str` and converts it to a rpython byte string.
+   PyPy3 this method will return an utf-8-nosg encoded result.
+
+.. py:function:: bytes_w(w_x)
+
+   Takes an application level :py:class:`bytes` (PyPy2 this equals `str`) and returns a rpython
+   byte string.
+
 .. py:function:: str_w(w_x)
 
+   **Deprecated. use text_w or bytes_w instead**
    If :py:obj:`w_x` is an application-level string, return an interpreter-level string.
    Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: unicode_w(w_x)
+
+   Takes an application level :py:class:`unicode` and return an
+   interpreter-level unicode string.  This method may disappear soon and
+   be replaced by :py:function:`text_w()`.
+
 .. py:function:: float_w(w_x)
 
    If :py:obj:`w_x` is an application-level float, integer or long, return an
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -32,9 +32,9 @@
 
 def create_entry_point(space, w_dict):
     if w_dict is not None: # for tests
-        w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
-        w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
-        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
+        w_entry_point = space.getitem(w_dict, space.newtext('entry_point'))
+        w_run_toplevel = space.getitem(w_dict, space.newtext('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.newtext('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
         hashfunc = space.config.objspace.hash
     else:
@@ -83,7 +83,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
             try:
@@ -91,7 +91,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
 
@@ -148,7 +148,7 @@
             if verbose:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
             if must_leave:
@@ -187,11 +187,11 @@
     def _pypy_execute_source(source, c_argument):
         try:
             w_globals = space.newdict(module=True)
-            space.setitem(w_globals, space.wrap('__builtins__'),
+            space.setitem(w_globals, space.newtext('__builtins__'),
                           space.builtin_modules['builtins'])
-            space.setitem(w_globals, space.wrap('c_argument'),
-                          space.wrap(c_argument))
-            space.appexec([space.wrap(source), w_globals], """(src, glob):
+            space.setitem(w_globals, space.newtext('c_argument'),
+                          space.newint(c_argument))
+            space.appexec([space.newtext(source), w_globals], """(src, glob):
                 import sys
                 stmt = compile(src, 'c callback', 'exec')
                 if not hasattr(sys, '_pypy_execute_source'):
@@ -202,7 +202,7 @@
         except OperationError as e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
 
@@ -333,7 +333,7 @@
         # obscure hack to stuff the translation options into the translated PyPy
         import pypy.module.sys
         options = make_dict(config)
-        wrapstr = 'space.wrap(%r)' % (options)
+        wrapstr = 'space.wrap(%r)' % (options) # import time
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -404,7 +404,7 @@
     i = 0
     for w_key in keys_w:
         try:
-            key = space.identifier_w(w_key)
+            key = space.text_w(w_key)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError,
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -269,8 +269,8 @@
             else:
                 w_key = space.newtuple([obj, space.w_float])
         elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
+            w_real = space.getattr(obj, space.newtext("real"))
+            w_imag = space.getattr(obj, space.newtext("imag"))
             real = space.float_w(w_real)
             imag = space.float_w(w_imag)
             real_negzero = (real == 0.0 and
@@ -369,7 +369,7 @@
         space = self.space
         consts_w = [space.w_None] * space.len_w(w_consts)
         w_iter = space.iter(w_consts)
-        first = space.wrap(0)
+        first = space.newint(0)
         while True:
             try:
                 w_key = space.next(w_iter)
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -13,7 +13,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -48,10 +48,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
-
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+        assert fields == []
+
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -67,14 +67,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -93,12 +93,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -114,7 +114,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -143,16 +143,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)
+            space.newtext(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
@@ -197,7 +197,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -232,7 +232,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -260,7 +260,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expression)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -296,7 +296,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -400,28 +400,28 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FunctionDef)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
         w_returns = self.returns.to_object(space) if self.returns is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('returns'), w_returns)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('returns'), w_returns)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -480,28 +480,28 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AsyncFunctionDef)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
         w_returns = self.returns.to_object(space) if self.returns is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('returns'), w_returns)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('returns'), w_returns)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -565,36 +565,36 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ClassDef)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.bases is None:
             bases_w = []
         else:
             bases_w = [node.to_object(space) for node in self.bases] # expr
         w_bases = space.newlist(bases_w)
-        space.setattr(w_node, space.wrap('bases'), w_bases)
+        space.setattr(w_node, space.newtext('bases'), w_bases)
         if self.keywords is None:
             keywords_w = []
         else:
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
-        space.setattr(w_node, space.wrap('keywords'), w_keywords)
+        space.setattr(w_node, space.newtext('keywords'), w_keywords)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -641,11 +641,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -684,11 +684,11 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -730,13 +730,13 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -776,15 +776,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AugAssign)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -838,25 +838,25 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_For)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -912,25 +912,25 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AsyncFor)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -984,23 +984,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_While)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1050,23 +1050,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_If)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1118,17 +1118,17 @@
         else:
             items_w = [node.to_object(space) for node in self.items] # withitem
         w_items = space.newlist(items_w)
-        space.setattr(w_node, space.wrap('items'), w_items)
+        space.setattr(w_node, space.newtext('items'), w_items)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1176,17 +1176,17 @@
         else:
             items_w = [node.to_object(space) for node in self.items] # withitem
         w_items = space.newlist(items_w)
-        space.setattr(w_node, space.wrap('items'), w_items)
+        space.setattr(w_node, space.newtext('items'), w_items)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1226,13 +1226,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
         w_exc = self.exc.to_object(space) if self.exc is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('exc'), w_exc)
+        space.setattr(w_node, space.newtext('exc'), w_exc)
         w_cause = self.cause.to_object(space) if self.cause is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('cause'), w_cause)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('cause'), w_cause)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1288,29 +1288,29 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.handlers is None:
             handlers_w = []
         else:
             handlers_w = [node.to_object(space) for node in self.handlers] # excepthandler
         w_handlers = space.newlist(handlers_w)
-        space.setattr(w_node, space.wrap('handlers'), w_handlers)
+        space.setattr(w_node, space.newtext('handlers'), w_handlers)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
         if self.finalbody is None:
             finalbody_w = []
         else:
             finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt
         w_finalbody = space.newlist(finalbody_w)
-        space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('finalbody'), w_finalbody)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1355,13 +1355,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Assert)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_msg = self.msg.to_object(space) if self.msg is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('msg'), w_msg)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('msg'), w_msg)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1404,11 +1404,11 @@
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1445,20 +1445,20 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = space.wrap(self.module.decode('utf-8')) if self.module is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('module'), w_module)
+        w_module = space.newtext_or_none(self.module)  # identifier
+        space.setattr(w_node, space.newtext('module'), w_module)
         if self.names is None:
             names_w = []
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_level = space.wrap(self.level)  # int
-        space.setattr(w_node, space.wrap('level'), w_level)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_level = space.newint(self.level)  # int
+        space.setattr(w_node, space.newtext('level'), w_level)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1468,7 +1468,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.str_or_None_w(w_module)
+        _module = space.text_or_None_w(w_module)
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -1496,13 +1496,13 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [space.wrap(node.decode('utf-8')) for node in self.names] # identifier
+            names_w = [space.newtext(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1536,13 +1536,13 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [space.wrap(node.decode('utf-8')) for node in self.names] # identifier
+            names_w = [space.newtext(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1575,11 +1575,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1610,10 +1610,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Pass)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1640,10 +1640,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Break)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1670,10 +1670,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Continue)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1781,17 +1781,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BoolOp)
         w_op = boolop_to_class[self.op - 1]().to_object(space)  # boolop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1831,15 +1831,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BinOp)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_right = self.right.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('right'), w_right)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('right'), w_right)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1882,13 +1882,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_UnaryOp)
         w_op = unaryop_to_class[self.op - 1]().to_object(space)  # unaryop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_operand = self.operand.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('operand'), w_operand)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('operand'), w_operand)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1928,13 +1928,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Lambda)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1976,15 +1976,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_IfExp)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_orelse = self.orelse.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2038,17 +2038,17 @@
         else:
             keys_w = [node.to_object(space) if node is not None else space.w_None for node in self.keys] # expr
         w_keys = space.newlist(keys_w)
-        space.setattr(w_node, space.wrap('keys'), w_keys)
+        space.setattr(w_node, space.newtext('keys'), w_keys)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2091,11 +2091,11 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2133,17 +2133,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ListComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2185,17 +2185,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_SetComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2239,19 +2239,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_DictComp)
         w_key = self.key.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('key'), w_key)
+        space.setattr(w_node, space.newtext('key'), w_key)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2297,17 +2297,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_GeneratorExp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2344,11 +2344,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Await)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2383,11 +2383,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2419,11 +2419,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_YieldFrom)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2463,23 +2463,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Compare)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         if self.ops is None:
             ops_w = []
         else:
             ops_w = [cmpop_to_class[node - 1]().to_object(space) for node in self.ops] # cmpop
         w_ops = space.newlist(ops_w)
-        space.setattr(w_node, space.wrap('ops'), w_ops)
+        space.setattr(w_node, space.newtext('ops'), w_ops)
         if self.comparators is None:
             comparators_w = []
         else:
             comparators_w = [node.to_object(space) for node in self.comparators] # expr
         w_comparators = space.newlist(comparators_w)
-        space.setattr(w_node, space.wrap('comparators'), w_comparators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('comparators'), w_comparators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2529,23 +2529,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Call)
         w_func = self.func.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('func'), w_func)
+        space.setattr(w_node, space.newtext('func'), w_func)
         if self.args is None:
             args_w = []
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.keywords is None:
             keywords_w = []
         else:
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
-        space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('keywords'), w_keywords)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2584,11 +2584,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Num)
         w_n = self.n  # object
-        space.setattr(w_node, space.wrap('n'), w_n)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('n'), w_n)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2621,11 +2621,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Str)
         w_s = self.s  # string
-        space.setattr(w_node, space.wrap('s'), w_s)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('s'), w_s)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2663,15 +2663,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FormattedValue)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_conversion = space.wrap(self.conversion)  # int
-        space.setattr(w_node, space.wrap('conversion'), w_conversion)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_conversion = space.newint(self.conversion)  # int
+        space.setattr(w_node, space.newtext('conversion'), w_conversion)
         w_format_spec = self.format_spec.to_object(space) if self.format_spec is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('format_spec'), w_format_spec)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('format_spec'), w_format_spec)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2716,11 +2716,11 @@
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2752,11 +2752,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Bytes)
         w_s = self.s  # bytes
-        space.setattr(w_node, space.wrap('s'), w_s)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('s'), w_s)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2789,11 +2789,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_NameConstant)
         w_value = self.value  # singleton
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2824,10 +2824,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Ellipsis)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2859,15 +2859,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Attribute)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_attr = space.wrap(self.attr.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('attr'), w_attr)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_attr = space.newtext(self.attr)  # identifier
+        space.setattr(w_node, space.newtext('attr'), w_attr)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2912,15 +2912,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Subscript)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         w_slice = self.slice.to_object(space)  # slice
-        space.setattr(w_node, space.wrap('slice'), w_slice)
+        space.setattr(w_node, space.newtext('slice'), w_slice)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2963,13 +2963,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Starred)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3006,14 +3006,14 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Name)
-        w_id = space.wrap(self.id.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('id'), w_id)
+        w_id = space.newtext(self.id)  # identifier
+        space.setattr(w_node, space.newtext('id'), w_id)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3059,13 +3059,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3110,13 +3110,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3152,11 +3152,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Const)
         w_obj = self.obj  # object
-        space.setattr(w_node, space.wrap('obj'), w_obj)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('obj'), w_obj)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3276,11 +3276,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
         w_lower = self.lower.to_object(space) if self.lower is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('lower'), w_lower)
+        space.setattr(w_node, space.newtext('lower'), w_lower)
         w_upper = self.upper.to_object(space) if self.upper is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('upper'), w_upper)
+        space.setattr(w_node, space.newtext('upper'), w_upper)
         w_step = self.step.to_object(space) if self.step is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('step'), w_step)
+        space.setattr(w_node, space.newtext('step'), w_step)
         return w_node
 
     @staticmethod
@@ -3318,7 +3318,7 @@
         else:
             dims_w = [node.to_object(space) for node in self.dims] # slice
         w_dims = space.newlist(dims_w)
-        space.setattr(w_node, space.wrap('dims'), w_dims)
+        space.setattr(w_node, space.newtext('dims'), w_dims)
         return w_node
 
     @staticmethod
@@ -3346,7 +3346,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Index)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
@@ -3687,15 +3687,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_comprehension)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.ifs is None:
             ifs_w = []
         else:
             ifs_w = [node.to_object(space) for node in self.ifs] # expr
         w_ifs = space.newlist(ifs_w)
-        space.setattr(w_node, space.wrap('ifs'), w_ifs)
+        space.setattr(w_node, space.newtext('ifs'), w_ifs)
         return w_node
 
     @staticmethod
@@ -3754,19 +3754,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
-        w_name = space.wrap(self.name.decode('utf-8')) if self.name is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        space.setattr(w_node, space.newtext('type'), w_type)
+        w_name = space.newtext_or_none(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3777,7 +3777,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _type = expr.from_object(space, w_type)
-        _name = space.str_or_None_w(w_name)
+        _name = space.text_or_None_w(w_name)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         _lineno = space.int_w(w_lineno)
@@ -3830,29 +3830,29 @@
         else:
             args_w = [node.to_object(space) for node in self.args] # arg
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         w_vararg = self.vararg.to_object(space) if self.vararg is not None else space.w_None  # arg
-        space.setattr(w_node, space.wrap('vararg'), w_vararg)
+        space.setattr(w_node, space.newtext('vararg'), w_vararg)
         if self.kwonlyargs is None:
             kwonlyargs_w = []
         else:
             kwonlyargs_w = [node.to_object(space) for node in self.kwonlyargs] # arg
         w_kwonlyargs = space.newlist(kwonlyargs_w)
-        space.setattr(w_node, space.wrap('kwonlyargs'), w_kwonlyargs)
+        space.setattr(w_node, space.newtext('kwonlyargs'), w_kwonlyargs)
         if self.kw_defaults is None:
             kw_defaults_w = []
         else:
             kw_defaults_w = [node.to_object(space) if node is not None else space.w_None for node in self.kw_defaults] # expr
         w_kw_defaults = space.newlist(kw_defaults_w)
-        space.setattr(w_node, space.wrap('kw_defaults'), w_kw_defaults)
+        space.setattr(w_node, space.newtext('kw_defaults'), w_kw_defaults)
         w_kwarg = self.kwarg.to_object(space) if self.kwarg is not None else space.w_None  # arg
-        space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
+        space.setattr(w_node, space.newtext('kwarg'), w_kwarg)
         if self.defaults is None:
             defaults_w = []
         else:
             defaults_w = [node.to_object(space) for node in self.defaults] # expr
         w_defaults = space.newlist(defaults_w)
-        space.setattr(w_node, space.wrap('defaults'), w_defaults)
+        space.setattr(w_node, space.newtext('defaults'), w_defaults)
         return w_node
 
     @staticmethod
@@ -3895,14 +3895,14 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_arg)
-        w_arg = space.wrap(self.arg.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_arg = space.newtext(self.arg)  # identifier
+        space.setattr(w_node, space.newtext('arg'), w_arg)
         w_annotation = self.annotation.to_object(space) if self.annotation is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('annotation'), w_annotation)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('annotation'), w_annotation)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3936,17 +3936,17 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.wrap(self.arg.decode('utf-8')) if self.arg is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_arg = space.newtext_or_none(self.arg)  # identifier
+        space.setattr(w_node, space.newtext('arg'), w_arg)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', True)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.str_or_None_w(w_arg)
+        _arg = space.text_or_None_w(w_arg)
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
@@ -3968,10 +3968,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_alias)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
-        w_asname = space.wrap(self.asname.decode('utf-8')) if self.asname is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('asname'), w_asname)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
+        w_asname = space.newtext_or_none(self.asname)  # identifier
+        space.setattr(w_node, space.newtext('asname'), w_asname)
         return w_node
 
     @staticmethod
@@ -3981,7 +3981,7 @@
         _name = space.identifier_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.str_or_None_w(w_asname)
+        _asname = space.text_or_None_w(w_asname)
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
@@ -4004,9 +4004,9 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_withitem)
         w_context_expr = self.context_expr.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
+        space.setattr(w_node, space.newtext('context_expr'), w_context_expr)
         w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
+        space.setattr(w_node, space.newtext('optional_vars'), w_optional_vars)
         return w_node
 
     @staticmethod
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1168,8 +1168,8 @@
                 raw = "0" + raw
         if negative:
             raw = "-" + raw
-        w_num_str = self.space.wrap(raw)
-        w_base = self.space.wrap(base)
+        w_num_str = self.space.newtext(raw)
+        w_base = self.space.newint(base)
         if raw[-1] in "jJ":
             tp = self.space.w_complex
             return self.space.call_function(tp, w_num_str)
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -313,7 +313,7 @@
 
     def _make_function(self, code, num_defaults=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
-        w_qualname = self.space.wrap((qualname or code.co_name).decode('utf-8'))
+        w_qualname = self.space.newtext(qualname or code.co_name)
         if code.co_freevars:
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
@@ -339,8 +339,8 @@
             if default:
                 kwonly = args.kwonlyargs[i]
                 assert isinstance(kwonly, ast.arg)
-                mangled = self.scope.mangle(kwonly.arg).decode('utf-8')
-                self.load_const(self.space.wrap(mangled))
+                mangled = self.scope.mangle(kwonly.arg)
+                self.load_const(self.space.newtext(mangled))
                 default.walkabout(self)
                 defaults += 1
         return defaults
@@ -375,8 +375,7 @@
         if l:
             if l > 65534:
                 self.error("too many annotations", func)
-            w_tup = space.newtuple([space.wrap(name.decode('utf-8'))
-                                    for name in names])
+            w_tup = space.newtuple([space.newtext(name) for name in names])
             self.load_const(w_tup)
             l += 1
         return l
@@ -438,7 +437,7 @@
         # 3. load a function (or closure) made from the code object
         self._make_function(code, qualname=qualname)
         # 4. load class name
-        self.load_const(self.space.wrap(cls.name.decode('utf-8')))
+        self.load_const(self.space.newtext(cls.name))
         # 5. generate the rest of the code for the call
         self._make_call(2, cls.bases, cls.keywords)
         # 6. apply decorators
@@ -808,7 +807,7 @@
         for alias in imp.names:
             assert isinstance(alias, ast.alias)
             level = 0
-            self.load_const(self.space.wrap(level))
+            self.load_const(self.space.newint(level))
             self.load_const(self.space.w_None)
             self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name)
             # If there's no asname then we store the root module.  If there is
@@ -846,12 +845,12 @@
                         self.error("not a chance", imp)
                     self.error("future feature %s is not defined" %
                                (alias.name,), imp)
-        self.load_const(space.wrap(imp.level))
+        self.load_const(space.newint(imp.level))
         names_w = [None]*len(imp.names)
         for i in range(len(imp.names)):
             alias = imp.names[i]
             assert isinstance(alias, ast.alias)
-            names_w[i] = space.wrap(alias.name.decode('utf-8'))
+            names_w[i] = space.newtext(alias.name)
         self.load_const(space.newtuple(names_w))
         if imp.module:
             mod_name = imp.module
@@ -1250,7 +1249,7 @@
 
     def visit_keyword(self, keyword):
         if keyword.arg is not None:
-            self.load_const(self.space.wrap(keyword.arg.decode('utf-8')))
+            self.load_const(self.space.newtext(keyword.arg))
         keyword.value.walkabout(self)
 
     def _make_call(self, n, # args already pushed
@@ -1308,7 +1307,7 @@
                     nsubkwargs += 1
                 elif nsubkwargs:
                     # A keyword argument and we already have a dict.
-                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
+                    self.load_const(self.space.newtext(kw.arg))
                     kw.value.walkabout(self)
                     nseen += 1
                 else:
@@ -1651,7 +1650,7 @@
         # ... and store it as __module__
         self.name_op("__module__", ast.Store)
         # store the qualname
-        w_qualname = self.space.wrap(self.qualname.decode("utf-8"))
+        w_qualname = self.space.newtext(self.qualname)
         self.load_const(w_qualname)
         self.name_op("__qualname__", ast.Store)
         # compile the body proper
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -19,10 +19,10 @@
 
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
-    w_msg = space.wrap(msg)
+    w_msg = space.newtext(msg)
     w_filename = space.wrap_fsdecoded(fn)
-    w_lineno = space.wrap(lineno)
-    w_offset = space.wrap(offset)
+    w_lineno = space.newint(lineno)
+    w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
 
 
@@ -127,6 +127,6 @@
         return name
 
     from pypy.module.unicodedata.interp_ucd import ucd
-    w_name = space.wrap(name.decode('utf-8'))
-    w_id = space.call_method(ucd, 'normalize', space.wrap('NFKC'), w_name)
-    return space.unicode_w(w_id).encode('utf-8')
+    w_name = space.newtext(name)
+    w_id = space.call_method(ucd, 'normalize', space.newtext('NFKC'), w_name)
+    return space.text_w(w_id)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -131,7 +131,7 @@
     return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
-    return space.wrap(not space.is_true(operand))
+    return space.newbool(not space.is_true(operand))
 
 
 binary_folders = {
@@ -222,7 +222,7 @@
                         break
                 else:
                     raise AssertionError("unknown unary operation")
-                w_minint = self.space.wrap(-sys.maxint - 1)
+                w_minint = self.space.newint(-sys.maxint - 1)
                 # This makes sure the result is an integer.
                 if self.space.eq_w(w_minint, w_const):
                     w_const = w_minint
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -130,15 +130,16 @@
     def get_value_converter(self, field, value):
         if field.type in self.data.simple_types:
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
-        elif field.type == "identifier":
-            wrapper = "space.wrap(%s.decode('utf-8'))" % (value,)
-            if field.opt:
-                wrapper += " if %s is not None else space.w_None" % (value,)
-            return wrapper
         elif field.type in ("object", "singleton", "string", "bytes"):
             return value
-        elif field.type in ("int", "bool"):
-            return "space.wrap(%s)" % (value,)
+        elif field.type == "bool":
+            return "space.newbool(%s)" % (value,)
+        elif field.type == "int":
+            return "space.newint(%s)" % (value,)
+        elif field.type == "identifier":
+            if field.opt:
+                return "space.newtext_or_none(%s)" % (value,)
+            return "space.newtext(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             allow_none = field.opt
@@ -160,7 +161,7 @@
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
-                return "space.str_or_None_w(%s)" % (value,)
+                return "space.text_or_None_w(%s)" % (value,)
             return "space.identifier_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
@@ -223,7 +224,7 @@
             wrapping_code = self.get_field_converter(field)
             for line in wrapping_code:
                 self.emit(line, 2)
-            self.emit("space.setattr(w_node, space.wrap(%r), w_%s)" % (
+            self.emit("space.setattr(w_node, space.newtext(%r), w_%s)" % (
                     str(field.name), field.name), 2)
         self.emit("return w_node", 2)
         self.emit("")
@@ -440,7 +441,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")

From pypy.commits at gmail.com  Thu Feb 16 14:57:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 11:57:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a6040c.52152e0a.d48ff.8de1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90172:697a8bbf4e66
Date: 2017-02-16 20:55 +0100
http://bitbucket.org/pypy/pypy/changeset/697a8bbf4e66/

Log:	hg merge default

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -159,7 +159,8 @@
 import libraries in the `out-of-line API mode`_. This is done by the following
 command::
 
-   PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py
+   cd pypy/goal
+   PYTHONPATH=../.. ./pypy-c ../tool/build_cffi_imports.py
 
 .. _`out-of-line API mode`: http://cffi.readthedocs.org/en/latest/overview.html#real-example-api-level-out-of-line
 

From pypy.commits at gmail.com  Thu Feb 16 14:57:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 11:57:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <58a6040f.c910190a.1e315.8f09@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90173:f6c56ea574e6
Date: 2017-02-16 20:55 +0100
http://bitbucket.org/pypy/pypy/changeset/f6c56ea574e6/

Log:	merge heads

diff too long, truncating to 2000 out of 25082 lines

diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -188,6 +188,7 @@
 
 .. py:function:: wrap(x)
 
+   **Deprecated! Eventually this method should disappear.**
    Returns a wrapped object that is a reference to the interpreter-level object
    :py:obj:`x`. This can be used either on simple immutable objects (integers,
    strings, etc) to create a new wrapped object, or on instances of :py:class:`W_Root`
@@ -196,6 +197,35 @@
    be directly exposed to application-level code in this way - functions, frames,
    code objects, etc.
 
+.. py:function:: newint(i)
+
+   Creates a wrapped object holding an integral value. `newint` creates an object
+   of type `W_IntObject`.
+
+.. py:function:: newlong(l)
+
+   Creates a wrapped object holding an integral value. The main difference to newint
+   is the type of the argument (which is rpython.rlib.rbigint.rbigint). On PyPy3 this
+   method will return an :py:class:`int` (PyPy2 it returns a :py:class:`long`).
+
+.. py:function:: newbytes(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`bytes` (both on PyPy2 and PyPy3).
+
+.. py:function:: newtext(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`str`.  On PyPy3 this will return a wrapped unicode
+   object. The object will hold a utf-8-nosg decoded value of `t`.
+   The "utf-8-nosg" codec used here is slightly different from the
+   "utf-8" implemented in Python 2 or Python 3: it is defined as utf-8
+   without any special handling of surrogate characters.  They are
+   encoded using the same three-bytes sequence that encodes any char in
+   the range from ``'\u0800'`` to ``'\uffff'``.
+
+   PyPy2 will return a bytestring object. No encoding/decoding steps will be applied.
+
 .. py:function:: newbool(b)
 
    Creates a wrapped :py:class:`bool` object from an :ref:`interpreter-level <interpreter-level>`
@@ -217,15 +247,18 @@
 
    Creates a new slice object.
 
-.. py:function:: newstring(asciilist)
+.. py:function:: newunicode(ustr)
 
-   Creates a string from a list of wrapped integers. Note that this may not be
-   a very useful method; usually you can just write ``space.wrap("mystring")``.
+   Creates a Unicode string from an rpython unicode string.
+   This method may disappear soon and be replaced by :py:function:`newutf8()`.
 
-.. py:function:: newunicode(codelist)
+.. py:function:: newutf8(bytestr)
 
-   Creates a Unicode string from a list of integers (code points).
+   Creates a Unicode string from an rpython byte string, decoded as
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
 
+Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
+`pypy/objspace/std/objspace.py`.
 
 Conversions from Application Level to Interpreter Level
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -258,11 +291,28 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: text_w(w_x)
+
+   Takes an application level :py:class:`str` and converts it to a rpython byte string.
+   PyPy3 this method will return an utf-8-nosg encoded result.
+
+.. py:function:: bytes_w(w_x)
+
+   Takes an application level :py:class:`bytes` (PyPy2 this equals `str`) and returns a rpython
+   byte string.
+
 .. py:function:: str_w(w_x)
 
+   **Deprecated. use text_w or bytes_w instead**
    If :py:obj:`w_x` is an application-level string, return an interpreter-level string.
    Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: unicode_w(w_x)
+
+   Takes an application level :py:class:`unicode` and return an
+   interpreter-level unicode string.  This method may disappear soon and
+   be replaced by :py:function:`text_w()`.
+
 .. py:function:: float_w(w_x)
 
    If :py:obj:`w_x` is an application-level float, integer or long, return an
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -32,9 +32,9 @@
 
 def create_entry_point(space, w_dict):
     if w_dict is not None: # for tests
-        w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
-        w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
-        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
+        w_entry_point = space.getitem(w_dict, space.newtext('entry_point'))
+        w_run_toplevel = space.getitem(w_dict, space.newtext('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.newtext('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
         hashfunc = space.config.objspace.hash
     else:
@@ -83,7 +83,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
             try:
@@ -91,7 +91,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
 
@@ -148,7 +148,7 @@
             if verbose:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
             if must_leave:
@@ -187,11 +187,11 @@
     def _pypy_execute_source(source, c_argument):
         try:
             w_globals = space.newdict(module=True)
-            space.setitem(w_globals, space.wrap('__builtins__'),
+            space.setitem(w_globals, space.newtext('__builtins__'),
                           space.builtin_modules['builtins'])
-            space.setitem(w_globals, space.wrap('c_argument'),
-                          space.wrap(c_argument))
-            space.appexec([space.wrap(source), w_globals], """(src, glob):
+            space.setitem(w_globals, space.newtext('c_argument'),
+                          space.newint(c_argument))
+            space.appexec([space.newtext(source), w_globals], """(src, glob):
                 import sys
                 stmt = compile(src, 'c callback', 'exec')
                 if not hasattr(sys, '_pypy_execute_source'):
@@ -202,7 +202,7 @@
         except OperationError as e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
-            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
 
@@ -333,7 +333,7 @@
         # obscure hack to stuff the translation options into the translated PyPy
         import pypy.module.sys
         options = make_dict(config)
-        wrapstr = 'space.wrap(%r)' % (options)
+        wrapstr = 'space.wrap(%r)' % (options) # import time
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -404,7 +404,7 @@
     i = 0
     for w_key in keys_w:
         try:
-            key = space.identifier_w(w_key)
+            key = space.text_w(w_key)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError,
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -269,8 +269,8 @@
             else:
                 w_key = space.newtuple([obj, space.w_float])
         elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
+            w_real = space.getattr(obj, space.newtext("real"))
+            w_imag = space.getattr(obj, space.newtext("imag"))
             real = space.float_w(w_real)
             imag = space.float_w(w_imag)
             real_negzero = (real == 0.0 and
@@ -369,7 +369,7 @@
         space = self.space
         consts_w = [space.w_None] * space.len_w(w_consts)
         w_iter = space.iter(w_consts)
-        first = space.wrap(0)
+        first = space.newint(0)
         while True:
             try:
                 w_key = space.next(w_iter)
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -13,7 +13,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -48,10 +48,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
-
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+        assert fields == []
+
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -67,14 +67,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -93,12 +93,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -114,7 +114,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -143,16 +143,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)
+            space.newtext(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
@@ -197,7 +197,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -232,7 +232,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -260,7 +260,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expression)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -296,7 +296,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -400,28 +400,28 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FunctionDef)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
         w_returns = self.returns.to_object(space) if self.returns is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('returns'), w_returns)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('returns'), w_returns)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -480,28 +480,28 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AsyncFunctionDef)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
         w_returns = self.returns.to_object(space) if self.returns is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('returns'), w_returns)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('returns'), w_returns)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -565,36 +565,36 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ClassDef)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.bases is None:
             bases_w = []
         else:
             bases_w = [node.to_object(space) for node in self.bases] # expr
         w_bases = space.newlist(bases_w)
-        space.setattr(w_node, space.wrap('bases'), w_bases)
+        space.setattr(w_node, space.newtext('bases'), w_bases)
         if self.keywords is None:
             keywords_w = []
         else:
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
-        space.setattr(w_node, space.wrap('keywords'), w_keywords)
+        space.setattr(w_node, space.newtext('keywords'), w_keywords)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -641,11 +641,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -684,11 +684,11 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -730,13 +730,13 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -776,15 +776,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AugAssign)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -838,25 +838,25 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_For)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -912,25 +912,25 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AsyncFor)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -984,23 +984,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_While)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1050,23 +1050,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_If)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1118,17 +1118,17 @@
         else:
             items_w = [node.to_object(space) for node in self.items] # withitem
         w_items = space.newlist(items_w)
-        space.setattr(w_node, space.wrap('items'), w_items)
+        space.setattr(w_node, space.newtext('items'), w_items)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1176,17 +1176,17 @@
         else:
             items_w = [node.to_object(space) for node in self.items] # withitem
         w_items = space.newlist(items_w)
-        space.setattr(w_node, space.wrap('items'), w_items)
+        space.setattr(w_node, space.newtext('items'), w_items)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1226,13 +1226,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
         w_exc = self.exc.to_object(space) if self.exc is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('exc'), w_exc)
+        space.setattr(w_node, space.newtext('exc'), w_exc)
         w_cause = self.cause.to_object(space) if self.cause is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('cause'), w_cause)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('cause'), w_cause)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1288,29 +1288,29 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.handlers is None:
             handlers_w = []
         else:
             handlers_w = [node.to_object(space) for node in self.handlers] # excepthandler
         w_handlers = space.newlist(handlers_w)
-        space.setattr(w_node, space.wrap('handlers'), w_handlers)
+        space.setattr(w_node, space.newtext('handlers'), w_handlers)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
         if self.finalbody is None:
             finalbody_w = []
         else:
             finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt
         w_finalbody = space.newlist(finalbody_w)
-        space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('finalbody'), w_finalbody)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1355,13 +1355,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Assert)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_msg = self.msg.to_object(space) if self.msg is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('msg'), w_msg)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('msg'), w_msg)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1404,11 +1404,11 @@
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1445,20 +1445,20 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = space.wrap(self.module.decode('utf-8')) if self.module is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('module'), w_module)
+        w_module = space.newtext_or_none(self.module)  # identifier
+        space.setattr(w_node, space.newtext('module'), w_module)
         if self.names is None:
             names_w = []
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_level = space.wrap(self.level)  # int
-        space.setattr(w_node, space.wrap('level'), w_level)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_level = space.newint(self.level)  # int
+        space.setattr(w_node, space.newtext('level'), w_level)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1468,7 +1468,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.str_or_None_w(w_module)
+        _module = space.text_or_None_w(w_module)
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -1496,13 +1496,13 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [space.wrap(node.decode('utf-8')) for node in self.names] # identifier
+            names_w = [space.newtext(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1536,13 +1536,13 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [space.wrap(node.decode('utf-8')) for node in self.names] # identifier
+            names_w = [space.newtext(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1575,11 +1575,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1610,10 +1610,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Pass)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1640,10 +1640,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Break)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1670,10 +1670,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Continue)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1781,17 +1781,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BoolOp)
         w_op = boolop_to_class[self.op - 1]().to_object(space)  # boolop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1831,15 +1831,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BinOp)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_right = self.right.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('right'), w_right)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('right'), w_right)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1882,13 +1882,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_UnaryOp)
         w_op = unaryop_to_class[self.op - 1]().to_object(space)  # unaryop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_operand = self.operand.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('operand'), w_operand)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('operand'), w_operand)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1928,13 +1928,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Lambda)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1976,15 +1976,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_IfExp)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_orelse = self.orelse.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2038,17 +2038,17 @@
         else:
             keys_w = [node.to_object(space) if node is not None else space.w_None for node in self.keys] # expr
         w_keys = space.newlist(keys_w)
-        space.setattr(w_node, space.wrap('keys'), w_keys)
+        space.setattr(w_node, space.newtext('keys'), w_keys)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2091,11 +2091,11 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2133,17 +2133,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ListComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2185,17 +2185,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_SetComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2239,19 +2239,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_DictComp)
         w_key = self.key.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('key'), w_key)
+        space.setattr(w_node, space.newtext('key'), w_key)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2297,17 +2297,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_GeneratorExp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2344,11 +2344,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Await)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2383,11 +2383,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2419,11 +2419,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_YieldFrom)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2463,23 +2463,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Compare)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         if self.ops is None:
             ops_w = []
         else:
             ops_w = [cmpop_to_class[node - 1]().to_object(space) for node in self.ops] # cmpop
         w_ops = space.newlist(ops_w)
-        space.setattr(w_node, space.wrap('ops'), w_ops)
+        space.setattr(w_node, space.newtext('ops'), w_ops)
         if self.comparators is None:
             comparators_w = []
         else:
             comparators_w = [node.to_object(space) for node in self.comparators] # expr
         w_comparators = space.newlist(comparators_w)
-        space.setattr(w_node, space.wrap('comparators'), w_comparators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('comparators'), w_comparators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2529,23 +2529,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Call)
         w_func = self.func.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('func'), w_func)
+        space.setattr(w_node, space.newtext('func'), w_func)
         if self.args is None:
             args_w = []
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.keywords is None:
             keywords_w = []
         else:
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
-        space.setattr(w_node, space.wrap('keywords'), w_keywords)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('keywords'), w_keywords)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2584,11 +2584,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Num)
         w_n = self.n  # object
-        space.setattr(w_node, space.wrap('n'), w_n)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('n'), w_n)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2621,11 +2621,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Str)
         w_s = self.s  # string
-        space.setattr(w_node, space.wrap('s'), w_s)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('s'), w_s)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2663,15 +2663,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FormattedValue)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_conversion = space.wrap(self.conversion)  # int
-        space.setattr(w_node, space.wrap('conversion'), w_conversion)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_conversion = space.newint(self.conversion)  # int
+        space.setattr(w_node, space.newtext('conversion'), w_conversion)
         w_format_spec = self.format_spec.to_object(space) if self.format_spec is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('format_spec'), w_format_spec)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('format_spec'), w_format_spec)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2716,11 +2716,11 @@
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2752,11 +2752,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Bytes)
         w_s = self.s  # bytes
-        space.setattr(w_node, space.wrap('s'), w_s)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('s'), w_s)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2789,11 +2789,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_NameConstant)
         w_value = self.value  # singleton
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2824,10 +2824,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Ellipsis)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2859,15 +2859,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Attribute)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_attr = space.wrap(self.attr.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('attr'), w_attr)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_attr = space.newtext(self.attr)  # identifier
+        space.setattr(w_node, space.newtext('attr'), w_attr)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2912,15 +2912,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Subscript)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         w_slice = self.slice.to_object(space)  # slice
-        space.setattr(w_node, space.wrap('slice'), w_slice)
+        space.setattr(w_node, space.newtext('slice'), w_slice)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2963,13 +2963,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Starred)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3006,14 +3006,14 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Name)
-        w_id = space.wrap(self.id.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('id'), w_id)
+        w_id = space.newtext(self.id)  # identifier
+        space.setattr(w_node, space.newtext('id'), w_id)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3059,13 +3059,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3110,13 +3110,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3152,11 +3152,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Const)
         w_obj = self.obj  # object
-        space.setattr(w_node, space.wrap('obj'), w_obj)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('obj'), w_obj)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3276,11 +3276,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
         w_lower = self.lower.to_object(space) if self.lower is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('lower'), w_lower)
+        space.setattr(w_node, space.newtext('lower'), w_lower)
         w_upper = self.upper.to_object(space) if self.upper is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('upper'), w_upper)
+        space.setattr(w_node, space.newtext('upper'), w_upper)
         w_step = self.step.to_object(space) if self.step is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('step'), w_step)
+        space.setattr(w_node, space.newtext('step'), w_step)
         return w_node
 
     @staticmethod
@@ -3318,7 +3318,7 @@
         else:
             dims_w = [node.to_object(space) for node in self.dims] # slice
         w_dims = space.newlist(dims_w)
-        space.setattr(w_node, space.wrap('dims'), w_dims)
+        space.setattr(w_node, space.newtext('dims'), w_dims)
         return w_node
 
     @staticmethod
@@ -3346,7 +3346,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Index)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
@@ -3687,15 +3687,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_comprehension)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.ifs is None:
             ifs_w = []
         else:
             ifs_w = [node.to_object(space) for node in self.ifs] # expr
         w_ifs = space.newlist(ifs_w)
-        space.setattr(w_node, space.wrap('ifs'), w_ifs)
+        space.setattr(w_node, space.newtext('ifs'), w_ifs)
         return w_node
 
     @staticmethod
@@ -3754,19 +3754,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
-        w_name = space.wrap(self.name.decode('utf-8')) if self.name is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        space.setattr(w_node, space.newtext('type'), w_type)
+        w_name = space.newtext_or_none(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3777,7 +3777,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _type = expr.from_object(space, w_type)
-        _name = space.str_or_None_w(w_name)
+        _name = space.text_or_None_w(w_name)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         _lineno = space.int_w(w_lineno)
@@ -3830,29 +3830,29 @@
         else:
             args_w = [node.to_object(space) for node in self.args] # arg
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         w_vararg = self.vararg.to_object(space) if self.vararg is not None else space.w_None  # arg
-        space.setattr(w_node, space.wrap('vararg'), w_vararg)
+        space.setattr(w_node, space.newtext('vararg'), w_vararg)
         if self.kwonlyargs is None:
             kwonlyargs_w = []
         else:
             kwonlyargs_w = [node.to_object(space) for node in self.kwonlyargs] # arg
         w_kwonlyargs = space.newlist(kwonlyargs_w)
-        space.setattr(w_node, space.wrap('kwonlyargs'), w_kwonlyargs)
+        space.setattr(w_node, space.newtext('kwonlyargs'), w_kwonlyargs)
         if self.kw_defaults is None:
             kw_defaults_w = []
         else:
             kw_defaults_w = [node.to_object(space) if node is not None else space.w_None for node in self.kw_defaults] # expr
         w_kw_defaults = space.newlist(kw_defaults_w)
-        space.setattr(w_node, space.wrap('kw_defaults'), w_kw_defaults)
+        space.setattr(w_node, space.newtext('kw_defaults'), w_kw_defaults)
         w_kwarg = self.kwarg.to_object(space) if self.kwarg is not None else space.w_None  # arg
-        space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
+        space.setattr(w_node, space.newtext('kwarg'), w_kwarg)
         if self.defaults is None:
             defaults_w = []
         else:
             defaults_w = [node.to_object(space) for node in self.defaults] # expr
         w_defaults = space.newlist(defaults_w)
-        space.setattr(w_node, space.wrap('defaults'), w_defaults)
+        space.setattr(w_node, space.newtext('defaults'), w_defaults)
         return w_node
 
     @staticmethod
@@ -3895,14 +3895,14 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_arg)
-        w_arg = space.wrap(self.arg.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_arg = space.newtext(self.arg)  # identifier
+        space.setattr(w_node, space.newtext('arg'), w_arg)
         w_annotation = self.annotation.to_object(space) if self.annotation is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('annotation'), w_annotation)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('annotation'), w_annotation)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3936,17 +3936,17 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.wrap(self.arg.decode('utf-8')) if self.arg is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_arg = space.newtext_or_none(self.arg)  # identifier
+        space.setattr(w_node, space.newtext('arg'), w_arg)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', True)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.str_or_None_w(w_arg)
+        _arg = space.text_or_None_w(w_arg)
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
@@ -3968,10 +3968,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_alias)
-        w_name = space.wrap(self.name.decode('utf-8'))  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
-        w_asname = space.wrap(self.asname.decode('utf-8')) if self.asname is not None else space.w_None  # identifier
-        space.setattr(w_node, space.wrap('asname'), w_asname)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
+        w_asname = space.newtext_or_none(self.asname)  # identifier
+        space.setattr(w_node, space.newtext('asname'), w_asname)
         return w_node
 
     @staticmethod
@@ -3981,7 +3981,7 @@
         _name = space.identifier_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.str_or_None_w(w_asname)
+        _asname = space.text_or_None_w(w_asname)
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
@@ -4004,9 +4004,9 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_withitem)
         w_context_expr = self.context_expr.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
+        space.setattr(w_node, space.newtext('context_expr'), w_context_expr)
         w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
+        space.setattr(w_node, space.newtext('optional_vars'), w_optional_vars)
         return w_node
 
     @staticmethod
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1168,8 +1168,8 @@
                 raw = "0" + raw
         if negative:
             raw = "-" + raw
-        w_num_str = self.space.wrap(raw)
-        w_base = self.space.wrap(base)
+        w_num_str = self.space.newtext(raw)
+        w_base = self.space.newint(base)
         if raw[-1] in "jJ":
             tp = self.space.w_complex
             return self.space.call_function(tp, w_num_str)
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -313,7 +313,7 @@
 
     def _make_function(self, code, num_defaults=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
-        w_qualname = self.space.wrap((qualname or code.co_name).decode('utf-8'))
+        w_qualname = self.space.newtext(qualname or code.co_name)
         if code.co_freevars:
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
@@ -339,8 +339,8 @@
             if default:
                 kwonly = args.kwonlyargs[i]
                 assert isinstance(kwonly, ast.arg)
-                mangled = self.scope.mangle(kwonly.arg).decode('utf-8')
-                self.load_const(self.space.wrap(mangled))
+                mangled = self.scope.mangle(kwonly.arg)
+                self.load_const(self.space.newtext(mangled))
                 default.walkabout(self)
                 defaults += 1
         return defaults
@@ -375,8 +375,7 @@
         if l:
             if l > 65534:
                 self.error("too many annotations", func)
-            w_tup = space.newtuple([space.wrap(name.decode('utf-8'))
-                                    for name in names])
+            w_tup = space.newtuple([space.newtext(name) for name in names])
             self.load_const(w_tup)
             l += 1
         return l
@@ -438,7 +437,7 @@
         # 3. load a function (or closure) made from the code object
         self._make_function(code, qualname=qualname)
         # 4. load class name
-        self.load_const(self.space.wrap(cls.name.decode('utf-8')))
+        self.load_const(self.space.newtext(cls.name))
         # 5. generate the rest of the code for the call
         self._make_call(2, cls.bases, cls.keywords)
         # 6. apply decorators
@@ -808,7 +807,7 @@
         for alias in imp.names:
             assert isinstance(alias, ast.alias)
             level = 0
-            self.load_const(self.space.wrap(level))
+            self.load_const(self.space.newint(level))
             self.load_const(self.space.w_None)
             self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name)
             # If there's no asname then we store the root module.  If there is
@@ -846,12 +845,12 @@
                         self.error("not a chance", imp)
                     self.error("future feature %s is not defined" %
                                (alias.name,), imp)
-        self.load_const(space.wrap(imp.level))
+        self.load_const(space.newint(imp.level))
         names_w = [None]*len(imp.names)
         for i in range(len(imp.names)):
             alias = imp.names[i]
             assert isinstance(alias, ast.alias)
-            names_w[i] = space.wrap(alias.name.decode('utf-8'))
+            names_w[i] = space.newtext(alias.name)
         self.load_const(space.newtuple(names_w))
         if imp.module:
             mod_name = imp.module
@@ -1250,7 +1249,7 @@
 
     def visit_keyword(self, keyword):
         if keyword.arg is not None:
-            self.load_const(self.space.wrap(keyword.arg.decode('utf-8')))
+            self.load_const(self.space.newtext(keyword.arg))
         keyword.value.walkabout(self)
 
     def _make_call(self, n, # args already pushed
@@ -1308,7 +1307,7 @@
                     nsubkwargs += 1
                 elif nsubkwargs:
                     # A keyword argument and we already have a dict.
-                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
+                    self.load_const(self.space.newtext(kw.arg))
                     kw.value.walkabout(self)
                     nseen += 1
                 else:
@@ -1651,7 +1650,7 @@
         # ... and store it as __module__
         self.name_op("__module__", ast.Store)
         # store the qualname
-        w_qualname = self.space.wrap(self.qualname.decode("utf-8"))
+        w_qualname = self.space.newtext(self.qualname)
         self.load_const(w_qualname)
         self.name_op("__qualname__", ast.Store)
         # compile the body proper
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -19,10 +19,10 @@
 
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
-    w_msg = space.wrap(msg)
+    w_msg = space.newtext(msg)
     w_filename = space.wrap_fsdecoded(fn)
-    w_lineno = space.wrap(lineno)
-    w_offset = space.wrap(offset)
+    w_lineno = space.newint(lineno)
+    w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
 
 
@@ -127,6 +127,6 @@
         return name
 
     from pypy.module.unicodedata.interp_ucd import ucd
-    w_name = space.wrap(name.decode('utf-8'))
-    w_id = space.call_method(ucd, 'normalize', space.wrap('NFKC'), w_name)
-    return space.unicode_w(w_id).encode('utf-8')
+    w_name = space.newtext(name)
+    w_id = space.call_method(ucd, 'normalize', space.newtext('NFKC'), w_name)
+    return space.text_w(w_id)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -131,7 +131,7 @@
     return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
-    return space.wrap(not space.is_true(operand))
+    return space.newbool(not space.is_true(operand))
 
 
 binary_folders = {
@@ -222,7 +222,7 @@
                         break
                 else:
                     raise AssertionError("unknown unary operation")
-                w_minint = self.space.wrap(-sys.maxint - 1)
+                w_minint = self.space.newint(-sys.maxint - 1)
                 # This makes sure the result is an integer.
                 if self.space.eq_w(w_minint, w_const):
                     w_const = w_minint
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -130,15 +130,16 @@
     def get_value_converter(self, field, value):
         if field.type in self.data.simple_types:
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
-        elif field.type == "identifier":
-            wrapper = "space.wrap(%s.decode('utf-8'))" % (value,)
-            if field.opt:
-                wrapper += " if %s is not None else space.w_None" % (value,)
-            return wrapper
         elif field.type in ("object", "singleton", "string", "bytes"):
             return value
-        elif field.type in ("int", "bool"):
-            return "space.wrap(%s)" % (value,)
+        elif field.type == "bool":
+            return "space.newbool(%s)" % (value,)
+        elif field.type == "int":
+            return "space.newint(%s)" % (value,)
+        elif field.type == "identifier":
+            if field.opt:
+                return "space.newtext_or_none(%s)" % (value,)
+            return "space.newtext(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             allow_none = field.opt
@@ -160,7 +161,7 @@
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
-                return "space.str_or_None_w(%s)" % (value,)
+                return "space.text_or_None_w(%s)" % (value,)
             return "space.identifier_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
@@ -223,7 +224,7 @@
             wrapping_code = self.get_field_converter(field)
             for line in wrapping_code:
                 self.emit(line, 2)
-            self.emit("space.setattr(w_node, space.wrap(%r), w_%s)" % (
+            self.emit("space.setattr(w_node, space.newtext(%r), w_%s)" % (
                     str(field.name), field.name), 2)
         self.emit("return w_node", 2)
         self.emit("")
@@ -440,7 +441,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")

From pypy.commits at gmail.com  Thu Feb 16 15:00:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 12:00:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a604f1.4b1b190a.22bd1.8d4b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90174:94d866dca83a
Date: 2017-02-16 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/94d866dca83a/

Log:	hg merge default


From pypy.commits at gmail.com  Thu Feb 16 14:56:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 16 Feb 2017 11:56:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-newtext: Close branch, ready to merge
Message-ID: <58a60407.53be190a.2c966.33fc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-newtext
Changeset: r90170:fb346ee5d28c
Date: 2017-02-16 20:51 +0100
http://bitbucket.org/pypy/pypy/changeset/fb346ee5d28c/

Log:	Close branch, ready to merge


From pypy.commits at gmail.com  Thu Feb 16 15:34:26 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 16 Feb 2017 12:34:26 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-2: merge default
Message-ID: <58a60cd2.18532e0a.7645f.8afa@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-2
Changeset: r90175:ec986a12d95a
Date: 2017-02-16 20:03 +0000
http://bitbucket.org/pypy/pypy/changeset/ec986a12d95a/

Log:	merge default

diff too long, truncating to 2000 out of 173321 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -77,3 +77,5 @@
 ^.hypothesis/
 ^release/
 ^rpython/_cache$
+
+pypy/module/cppyy/.+/*\.pcm
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2016
+PyPy Copyright holders 2003-2017
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,423 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:

From pypy.commits at gmail.com  Thu Feb 16 17:00:33 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 16 Feb 2017 14:00:33 -0800 (PST)
Subject: [pypy-commit] pypy default: Add failing test,
 but prevent PyThreadState_GetDict() from crashing when it doesn't
 have the GIL
Message-ID: <58a62101.15502e0a.10271.922e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90176:0afdf7591078
Date: 2017-02-16 21:59 +0000
http://bitbucket.org/pypy/pypy/changeset/0afdf7591078/

Log:	Add failing test, but prevent PyThreadState_GetDict() from crashing
	when it doesn't have the GIL

diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -180,7 +180,10 @@
     Previously this could only be called when a current thread is active, and NULL
     meant that an exception was raised."""
     state = space.fromcache(InterpreterState)
-    return state.get_thread_state(space).c_dict
+    ts = state.get_thread_state(space)
+    if not ts:
+        return lltype.nullptr(PyObject.TO)
+    return ts.c_dict
 
 @cpython_api([PyThreadState], PyThreadState, error=CANNOT_FAIL)
 def PyThreadState_Swap(space, tstate):
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -35,7 +35,7 @@
                 PyEval_InitThreads();
                 state0 = PyGILState_Ensure(); /* hangs here */
                 if (val != 0)
-                { 
+                {
                     state1 = PyGILState_Ensure();
                     PyGILState_Release(state1);
                 }
@@ -141,6 +141,40 @@
         res = module.bounce()
         assert res == 3
 
+    def test_thread_and_gil(self):
+        module = self.import_extension('foo', [
+            ("bounce", "METH_NOARGS",
+            """
+            PyThreadState * tstate;
+            if (PyEval_ThreadsInitialized() == 0)
+            {
+            PyEval_InitThreads();
+            }
+            tstate = PyEval_SaveThread();
+            if (tstate == NULL) {
+                return PyLong_FromLong(0);
+            }
+            PyObject* dict = PyThreadState_GetDict();
+            if (dict != NULL) {
+            return PyLong_FromLong(1);
+            }
+            PyGILState_STATE gilstate = PyGILState_Ensure();
+            dict = PyThreadState_GetDict();
+            if (dict == NULL) {
+            return PyLong_FromLong(2);
+            }
+            PyGILState_Release(gilstate);
+            PyEval_RestoreThread(tstate);
+
+            if (PyThreadState_Get() != tstate) {
+                return PyLong_FromLong(3);
+            }
+
+            return PyLong_FromLong(4);
+            """)])
+        res = module.bounce()
+        assert res == 4
+
     def test_threadsinitialized(self):
         module = self.import_extension('foo', [
                 ("test", "METH_NOARGS",
@@ -151,8 +185,8 @@
         res = module.test()
         print "got", res
         assert res in (0, 1)
-        
-        
+
+
 class AppTestState(AppTestCpythonExtensionBase):
 
     def test_frame_tstate_tracing(self):

From pypy.commits at gmail.com  Sat Feb 18 11:02:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Feb 2017 08:02:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test, but very unsure about the point
Message-ID: <58a87026.4b1b190a.22bd1.ea3a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90177:2fa452b4957e
Date: 2017-02-18 13:21 +0100
http://bitbucket.org/pypy/pypy/changeset/2fa452b4957e/

Log:	fix test, but very unsure about the point

diff --git a/pypy/module/_locale/test/test_locale.py b/pypy/module/_locale/test/test_locale.py
--- a/pypy/module/_locale/test/test_locale.py
+++ b/pypy/module/_locale/test/test_locale.py
@@ -115,8 +115,8 @@
         ucase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
         _locale.setlocale(_locale.LC_ALL, self.language_utf8)
-        assert string.lowercase == lcase
-        assert string.uppercase == ucase
+        assert string.ascii_lowercase == lcase
+        assert string.ascii_uppercase == ucase
 
         _locale.setlocale(_locale.LC_ALL, self.language_en)
 

From pypy.commits at gmail.com  Sat Feb 18 11:02:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Feb 2017 08:02:51 -0800 (PST)
Subject: [pypy-commit] pypy default: Try to make at least the utf-8
 encoding/decoding functions elidable.
Message-ID: <58a8702b.51072e0a.d002.7b41@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90178:bfe85246978d
Date: 2017-02-18 15:54 +0100
http://bitbucket.org/pypy/pypy/changeset/bfe85246978d/

Log:	Try to make at least the utf-8 encoding/decoding functions elidable.
	This is messy for several reasons. Now if you call the functions
	with an errorhandler that can't be used inside an @elidable, then
	you have to fix the calls to invoke another version.

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -442,7 +442,10 @@
     if errors is None:
         errors = 'strict'
     state = space.fromcache(CodecState)
-    result = runicode.unicode_encode_utf_8(
+    # NB. can't call unicode_encode_utf_8() directly because that's
+    # an @elidable function nowadays.  Instead, we need the _impl().
+    # (The problem is the errorhandler, which calls arbitrary Python.)
+    result = runicode.unicode_encode_utf_8_impl(
         uni, len(uni), errors, state.encode_error_handler,
         allow_surrogates=True)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
@@ -454,7 +457,10 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    result, consumed = runicode.str_decode_utf_8(
+    # NB. can't call str_decode_utf_8() directly because that's
+    # an @elidable function nowadays.  Instead, we need the _impl().
+    # (The problem is the errorhandler, which calls arbitrary Python.)
+    result, consumed = runicode.str_decode_utf_8_impl(
         string, len(string), errors,
         final, state.decode_error_handler,
         allow_surrogates=True)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -3,8 +3,10 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
+from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import jit
+from rpython.rlib.nonconst import NonConstant
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -127,15 +129,24 @@
     4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0  # F0-F4 - F5-FF
 ]
 
+# if you can't use the @elidable version, call str_decode_utf_8_impl()
+# directly
+ at jit.elidable
 def str_decode_utf_8(s, size, errors, final=False,
                      errorhandler=None, allow_surrogates=allow_surrogate_by_default):
     if errorhandler is None:
         errorhandler = default_unicode_error_decode
-    result = UnicodeBuilder(size)
-    pos = str_decode_utf_8_impl(s, size, errors, final, errorhandler,
-                                 allow_surrogates=allow_surrogates,
-                                 result=result)
-    return result.build(), pos
+    # NB. a bit messy because rtyper/rstr.py also calls the same
+    # function.  Make sure we annotate for the args it passes, too
+    if NonConstant(False):
+        s = '?????'
+        size = 12345
+        errors = 'strict'
+        final = True
+        errorhandler = default_unicode_error_decode
+        allow_surrogates = False
+    return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
+                                     allow_surrogates=allow_surrogates)
 
 def _invalid_cont_byte(ordch):
     return ordch>>6 != 0x2    # 0b10
@@ -157,14 +168,12 @@
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
-# note: this specialize() is here for rtyper/rstr.py, which calls this
-# function too but with its own fixed errorhandler
- at specialize.arg_or_var(4)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
-                          allow_surrogates, result):
+                          allow_surrogates):
     if size == 0:
-        return 0
+        return u'', 0
 
+    result = UnicodeBuilder(size)
     pos = 0
     while pos < size:
         ordch1 = ord(s[pos])
@@ -316,7 +325,9 @@
                 result.append(unichr(0xDC00 + (c & 0x03FF)))
             pos += 4
 
-    return pos
+    return result.build(), pos
+str_decode_utf_8_elidable = jit.elidable(
+    func_with_new_name(str_decode_utf_8_impl, "str_decode_utf_8_elidable"))
 
 def _encodeUCS4(result, ch):
     # Encode UCS4 Unicode ordinals
@@ -325,6 +336,9 @@
     result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
     result.append((chr((0x80 | (ch & 0x3f)))))
 
+# if you can't use the @elidable version, call unicode_encode_utf_8_impl()
+# directly
+ at jit.elidable
 def unicode_encode_utf_8(s, size, errors, errorhandler=None,
                          allow_surrogates=allow_surrogate_by_default):
     # In this function, allow_surrogates can be:
@@ -339,12 +353,17 @@
     #
     if errorhandler is None:
         errorhandler = default_unicode_error_encode
-    return unicode_encode_utf_8_impl(s, size, errors, errorhandler,
-                                     allow_surrogates=allow_surrogates)
+    # NB. a bit messy because rtyper/rstr.py also calls the same
+    # function.  Make sure we annotate for the args it passes, too
+    if NonConstant(False):
+        s = u'?????'
+        size = 12345
+        errors = 'strict'
+        errorhandler = default_unicode_error_encode
+        allow_surrogates = False
+    return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
+                                         allow_surrogates=allow_surrogates)
 
-# note: this specialize() is here for rtyper/rstr.py, which calls this
-# function too but with its own fixed errorhandler
- at specialize.arg_or_var(3)
 def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                               allow_surrogates=False):
     assert(size >= 0)
@@ -400,6 +419,9 @@
             else:
                 _encodeUCS4(result, ch)
     return result.build()
+unicode_encode_utf_8_elidable = jit.elidable(
+    func_with_new_name(unicode_encode_utf_8_impl,
+                       "unicode_encode_utf_8_elidable"))
 
 def unicode_encode_utf8sp(s, size):
     # Surrogate-preserving utf-8 encoding.  Any surrogate character
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -9,7 +9,6 @@
 from rpython.rtyper.rfloat import FloatRepr
 from rpython.tool.pairtype import pairtype, pair
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.rstring import UnicodeBuilder
 
 
 class AbstractStringRepr(Repr):
@@ -18,31 +17,21 @@
         Repr.__init__(self, *args)
         self.rstr_decode_utf_8 = None
 
-    def ensure_ll_decode_utf8(self):
-        from rpython.rlib.runicode import str_decode_utf_8_impl
-        self.rstr_decode_utf_8 = func_with_new_name(str_decode_utf_8_impl,
-                                                    'rstr_decode_utf_8_impl')
-
     @jit.elidable
     def ll_decode_utf8(self, llvalue):
         from rpython.rtyper.annlowlevel import hlstr
+        from rpython.rlib import runicode
         value = hlstr(llvalue)
         assert value is not None
-        result = UnicodeBuilder(len(value))
-        self.rstr_decode_utf_8(
-            value, len(value), 'strict', final=True,
-            errorhandler=self.ll_raise_unicode_exception_decode,
-            allow_surrogates=False, result=result)
+        errorhandler = runicode.default_unicode_error_decode
+        u, pos = runicode.str_decode_utf_8_elidable(
+            value, len(value), 'strict', True, errorhandler, False)
         # XXX should it really be 'allow_surrogates=False'?  In RPython,
         # unicode.decode('utf-8') happily accepts surrogates.  This
         # makes it hard to test untranslated (it's the cause of a
         # failure in lib-python's test_warnings on PyPy3, for example)
-        return self.ll.llunicode(result.build())
-
-    @staticmethod
-    def ll_raise_unicode_exception_decode(errors, encoding, msg, s,
-                                       startingpos, endingpos):
-        raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
+        # XXX maybe the whole ''.decode('utf-8') should be not RPython.
+        return self.ll.llunicode(u)
 
     def _str_reprs(self, hop):
         return hop.args_r[0].repr, hop.args_r[1].repr
@@ -351,7 +340,6 @@
         elif encoding == 'latin-1':
             return hop.gendirectcall(self.ll_decode_latin1, v_self)
         elif encoding == 'utf-8':
-            self.ensure_ll_decode_utf8()
             return hop.gendirectcall(self.ll_decode_utf8, v_self)
         else:
             raise TyperError("encoding %s not implemented" % (encoding, ))
@@ -394,11 +382,6 @@
         AbstractStringRepr.__init__(self, *args)
         self.runicode_encode_utf_8 = None
 
-    def ensure_ll_encode_utf8(self):
-        from rpython.rlib.runicode import unicode_encode_utf_8_impl
-        self.runicode_encode_utf_8 = func_with_new_name(
-            unicode_encode_utf_8_impl, 'runicode_encode_utf_8')
-
     def rtype_method_upper(self, hop):
         raise TyperError("Cannot do toupper on unicode string")
 
@@ -408,19 +391,15 @@
     @jit.elidable
     def ll_encode_utf8(self, ll_s):
         from rpython.rtyper.annlowlevel import hlunicode
+        from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
-        bytes = self.runicode_encode_utf_8(
+        errorhandler = runicode.default_unicode_error_encode
+        bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict',
-            errorhandler=self.ll_raise_unicode_exception_encode,
-            allow_surrogates=False)
+            errorhandler=errorhandler, allow_surrogates=False)
         return self.ll.llstr(bytes)
 
-    @staticmethod
-    def ll_raise_unicode_exception_encode(errors, encoding, msg, u,
-                                          startingpos, endingpos):
-        raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
-
     def rtype_method_encode(self, hop):
         if not hop.args_s[1].is_constant():
             raise TyperError("encoding must be constant")
@@ -436,7 +415,6 @@
         elif encoding == "latin-1":
             return hop.gendirectcall(self.ll_encode_latin1, v_self)
         elif encoding == 'utf-8':
-            self.ensure_ll_encode_utf8()
             return hop.gendirectcall(self.ll_encode_utf8, v_self)
         else:
             raise TyperError("encoding %s not implemented" % (encoding, ))
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -188,7 +188,7 @@
             else:
                 errors = 'foo'
             # the annotation of y is SomeUnicodeString(can_be_None=False)
-            y, _ = str_decode_utf_8(x, len(x), errors, errorhandler)
+            y, _ = str_decode_utf_8(x, len(x), errors, errorhandler=errorhandler)
             return x.decode('utf-8') + y
 
         assert self.ll_to_string(self.interpret(f, [1])) == f(1)

From pypy.commits at gmail.com  Sat Feb 18 11:02:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Feb 2017 08:02:53 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58a8702d.52512e0a.aaaec.ee0c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90179:7b6ea7df79b3
Date: 2017-02-18 17:00 +0100
http://bitbucket.org/pypy/pypy/changeset/7b6ea7df79b3/

Log:	merge heads

diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -180,7 +180,10 @@
     Previously this could only be called when a current thread is active, and NULL
     meant that an exception was raised."""
     state = space.fromcache(InterpreterState)
-    return state.get_thread_state(space).c_dict
+    ts = state.get_thread_state(space)
+    if not ts:
+        return lltype.nullptr(PyObject.TO)
+    return ts.c_dict
 
 @cpython_api([PyThreadState], PyThreadState, error=CANNOT_FAIL)
 def PyThreadState_Swap(space, tstate):
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -35,7 +35,7 @@
                 PyEval_InitThreads();
                 state0 = PyGILState_Ensure(); /* hangs here */
                 if (val != 0)
-                { 
+                {
                     state1 = PyGILState_Ensure();
                     PyGILState_Release(state1);
                 }
@@ -141,6 +141,40 @@
         res = module.bounce()
         assert res == 3
 
+    def test_thread_and_gil(self):
+        module = self.import_extension('foo', [
+            ("bounce", "METH_NOARGS",
+            """
+            PyThreadState * tstate;
+            if (PyEval_ThreadsInitialized() == 0)
+            {
+            PyEval_InitThreads();
+            }
+            tstate = PyEval_SaveThread();
+            if (tstate == NULL) {
+                return PyLong_FromLong(0);
+            }
+            PyObject* dict = PyThreadState_GetDict();
+            if (dict != NULL) {
+            return PyLong_FromLong(1);
+            }
+            PyGILState_STATE gilstate = PyGILState_Ensure();
+            dict = PyThreadState_GetDict();
+            if (dict == NULL) {
+            return PyLong_FromLong(2);
+            }
+            PyGILState_Release(gilstate);
+            PyEval_RestoreThread(tstate);
+
+            if (PyThreadState_Get() != tstate) {
+                return PyLong_FromLong(3);
+            }
+
+            return PyLong_FromLong(4);
+            """)])
+        res = module.bounce()
+        assert res == 4
+
     def test_threadsinitialized(self):
         module = self.import_extension('foo', [
                 ("test", "METH_NOARGS",
@@ -151,8 +185,8 @@
         res = module.test()
         print "got", res
         assert res in (0, 1)
-        
-        
+
+
 class AppTestState(AppTestCpythonExtensionBase):
 
     def test_frame_tstate_tracing(self):

From pypy.commits at gmail.com  Sat Feb 18 11:02:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Feb 2017 08:02:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a87033.ce18190a.d70c9.0760@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90180:14e732427366
Date: 2017-02-18 17:01 +0100
http://bitbucket.org/pypy/pypy/changeset/14e732427366/

Log:	hg merge default

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -707,7 +707,10 @@
     if errors is None:
         errors = 'strict'
     state = space.fromcache(CodecState)
-    result = runicode.unicode_encode_utf_8(
+    # NB. can't call unicode_encode_utf_8() directly because that's
+    # an @elidable function nowadays.  Instead, we need the _impl().
+    # (The problem is the errorhandler, which calls arbitrary Python.)
+    result = runicode.unicode_encode_utf_8_impl(
         uni, len(uni), errors, state.encode_error_handler,
         allow_surrogates=False)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
@@ -719,7 +722,10 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    result, consumed = runicode.str_decode_utf_8(
+    # NB. can't call str_decode_utf_8() directly because that's
+    # an @elidable function nowadays.  Instead, we need the _impl().
+    # (The problem is the errorhandler, which calls arbitrary Python.)
+    result, consumed = runicode.str_decode_utf_8_impl(
         string, len(string), errors,
         final, state.decode_error_handler,
         allow_surrogates=False)
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -184,7 +184,10 @@
     Previously this could only be called when a current thread is active, and NULL
     meant that an exception was raised."""
     state = space.fromcache(InterpreterState)
-    return state.get_thread_state(space).c_dict
+    ts = state.get_thread_state(space)
+    if not ts:
+        return lltype.nullptr(PyObject.TO)
+    return ts.c_dict
 
 @cpython_api([PyThreadState], PyThreadState, error=CANNOT_FAIL)
 def PyThreadState_Swap(space, tstate):
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -137,6 +137,40 @@
         res = module.bounce()
         assert res == 3
 
+    def test_thread_and_gil(self):
+        module = self.import_extension('foo', [
+            ("bounce", "METH_NOARGS",
+            """
+            PyThreadState * tstate;
+            if (PyEval_ThreadsInitialized() == 0)
+            {
+            PyEval_InitThreads();
+            }
+            tstate = PyEval_SaveThread();
+            if (tstate == NULL) {
+                return PyLong_FromLong(0);
+            }
+            PyObject* dict = PyThreadState_GetDict();
+            if (dict != NULL) {
+            return PyLong_FromLong(1);
+            }
+            PyGILState_STATE gilstate = PyGILState_Ensure();
+            dict = PyThreadState_GetDict();
+            if (dict == NULL) {
+            return PyLong_FromLong(2);
+            }
+            PyGILState_Release(gilstate);
+            PyEval_RestoreThread(tstate);
+
+            if (PyThreadState_Get() != tstate) {
+                return PyLong_FromLong(3);
+            }
+
+            return PyLong_FromLong(4);
+            """)])
+        res = module.bounce()
+        assert res == 4
+
     def test_threadsinitialized(self):
         module = self.import_extension('foo', [
                 ("test", "METH_NOARGS",
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -3,8 +3,10 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
+from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import jit
+from rpython.rlib.nonconst import NonConstant
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -127,15 +129,24 @@
     4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0  # F0-F4 - F5-FF
 ]
 
+# if you can't use the @elidable version, call str_decode_utf_8_impl()
+# directly
+ at jit.elidable
 def str_decode_utf_8(s, size, errors, final=False,
                      errorhandler=None, allow_surrogates=allow_surrogate_by_default):
     if errorhandler is None:
         errorhandler = default_unicode_error_decode
-    result = UnicodeBuilder(size)
-    pos = str_decode_utf_8_impl(s, size, errors, final, errorhandler,
-                                 allow_surrogates=allow_surrogates,
-                                 result=result)
-    return result.build(), pos
+    # NB. a bit messy because rtyper/rstr.py also calls the same
+    # function.  Make sure we annotate for the args it passes, too
+    if NonConstant(False):
+        s = '?????'
+        size = 12345
+        errors = 'strict'
+        final = True
+        errorhandler = default_unicode_error_decode
+        allow_surrogates = False
+    return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
+                                     allow_surrogates=allow_surrogates)
 
 def _invalid_cont_byte(ordch):
     return ordch>>6 != 0x2    # 0b10
@@ -157,14 +168,12 @@
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
-# note: this specialize() is here for rtyper/rstr.py, which calls this
-# function too but with its own fixed errorhandler
- at specialize.arg_or_var(4)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
-                          allow_surrogates, result):
+                          allow_surrogates):
     if size == 0:
-        return 0
+        return u'', 0
 
+    result = UnicodeBuilder(size)
     pos = 0
     while pos < size:
         ordch1 = ord(s[pos])
@@ -316,7 +325,9 @@
                 result.append(unichr(0xDC00 + (c & 0x03FF)))
             pos += 4
 
-    return pos
+    return result.build(), pos
+str_decode_utf_8_elidable = jit.elidable(
+    func_with_new_name(str_decode_utf_8_impl, "str_decode_utf_8_elidable"))
 
 def _encodeUCS4(result, ch):
     # Encode UCS4 Unicode ordinals
@@ -325,6 +336,9 @@
     result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
     result.append((chr((0x80 | (ch & 0x3f)))))
 
+# if you can't use the @elidable version, call unicode_encode_utf_8_impl()
+# directly
+ at jit.elidable
 def unicode_encode_utf_8(s, size, errors, errorhandler=None,
                          allow_surrogates=allow_surrogate_by_default):
     # In this function, allow_surrogates can be:
@@ -339,12 +353,17 @@
     #
     if errorhandler is None:
         errorhandler = default_unicode_error_encode
-    return unicode_encode_utf_8_impl(s, size, errors, errorhandler,
-                                     allow_surrogates=allow_surrogates)
+    # NB. a bit messy because rtyper/rstr.py also calls the same
+    # function.  Make sure we annotate for the args it passes, too
+    if NonConstant(False):
+        s = u'?????'
+        size = 12345
+        errors = 'strict'
+        errorhandler = default_unicode_error_encode
+        allow_surrogates = False
+    return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
+                                         allow_surrogates=allow_surrogates)
 
-# note: this specialize() is here for rtyper/rstr.py, which calls this
-# function too but with its own fixed errorhandler
- at specialize.arg_or_var(3)
 def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                               allow_surrogates=False):
     assert(size >= 0)
@@ -400,6 +419,9 @@
             else:
                 _encodeUCS4(result, ch)
     return result.build()
+unicode_encode_utf_8_elidable = jit.elidable(
+    func_with_new_name(unicode_encode_utf_8_impl,
+                       "unicode_encode_utf_8_elidable"))
 
 def unicode_encode_utf8sp(s, size):
     # Surrogate-preserving utf-8 encoding.  Any surrogate character
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -9,7 +9,6 @@
 from rpython.rtyper.rfloat import FloatRepr
 from rpython.tool.pairtype import pairtype, pair
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.rstring import UnicodeBuilder
 
 
 class AbstractStringRepr(Repr):
@@ -18,31 +17,21 @@
         Repr.__init__(self, *args)
         self.rstr_decode_utf_8 = None
 
-    def ensure_ll_decode_utf8(self):
-        from rpython.rlib.runicode import str_decode_utf_8_impl
-        self.rstr_decode_utf_8 = func_with_new_name(str_decode_utf_8_impl,
-                                                    'rstr_decode_utf_8_impl')
-
     @jit.elidable
     def ll_decode_utf8(self, llvalue):
         from rpython.rtyper.annlowlevel import hlstr
+        from rpython.rlib import runicode
         value = hlstr(llvalue)
         assert value is not None
-        result = UnicodeBuilder(len(value))
-        self.rstr_decode_utf_8(
-            value, len(value), 'strict', final=True,
-            errorhandler=self.ll_raise_unicode_exception_decode,
-            allow_surrogates=False, result=result)
+        errorhandler = runicode.default_unicode_error_decode
+        u, pos = runicode.str_decode_utf_8_elidable(
+            value, len(value), 'strict', True, errorhandler, False)
         # XXX should it really be 'allow_surrogates=False'?  In RPython,
         # unicode.decode('utf-8') happily accepts surrogates.  This
         # makes it hard to test untranslated (it's the cause of a
         # failure in lib-python's test_warnings on PyPy3, for example)
-        return self.ll.llunicode(result.build())
-
-    @staticmethod
-    def ll_raise_unicode_exception_decode(errors, encoding, msg, s,
-                                       startingpos, endingpos):
-        raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
+        # XXX maybe the whole ''.decode('utf-8') should be not RPython.
+        return self.ll.llunicode(u)
 
     def _str_reprs(self, hop):
         return hop.args_r[0].repr, hop.args_r[1].repr
@@ -351,7 +340,6 @@
         elif encoding == 'latin-1':
             return hop.gendirectcall(self.ll_decode_latin1, v_self)
         elif encoding == 'utf-8':
-            self.ensure_ll_decode_utf8()
             return hop.gendirectcall(self.ll_decode_utf8, v_self)
         else:
             raise TyperError("encoding %s not implemented" % (encoding, ))
@@ -394,11 +382,6 @@
         AbstractStringRepr.__init__(self, *args)
         self.runicode_encode_utf_8 = None
 
-    def ensure_ll_encode_utf8(self):
-        from rpython.rlib.runicode import unicode_encode_utf_8_impl
-        self.runicode_encode_utf_8 = func_with_new_name(
-            unicode_encode_utf_8_impl, 'runicode_encode_utf_8')
-
     def rtype_method_upper(self, hop):
         raise TyperError("Cannot do toupper on unicode string")
 
@@ -408,19 +391,15 @@
     @jit.elidable
     def ll_encode_utf8(self, ll_s):
         from rpython.rtyper.annlowlevel import hlunicode
+        from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
-        bytes = self.runicode_encode_utf_8(
+        errorhandler = runicode.default_unicode_error_encode
+        bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict',
-            errorhandler=self.ll_raise_unicode_exception_encode,
-            allow_surrogates=False)
+            errorhandler=errorhandler, allow_surrogates=False)
         return self.ll.llstr(bytes)
 
-    @staticmethod
-    def ll_raise_unicode_exception_encode(errors, encoding, msg, u,
-                                          startingpos, endingpos):
-        raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
-
     def rtype_method_encode(self, hop):
         if not hop.args_s[1].is_constant():
             raise TyperError("encoding must be constant")
@@ -436,7 +415,6 @@
         elif encoding == "latin-1":
             return hop.gendirectcall(self.ll_encode_latin1, v_self)
         elif encoding == 'utf-8':
-            self.ensure_ll_encode_utf8()
             return hop.gendirectcall(self.ll_encode_utf8, v_self)
         else:
             raise TyperError("encoding %s not implemented" % (encoding, ))
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -188,7 +188,7 @@
             else:
                 errors = 'foo'
             # the annotation of y is SomeUnicodeString(can_be_None=False)
-            y, _ = str_decode_utf_8(x, len(x), errors, errorhandler)
+            y, _ = str_decode_utf_8(x, len(x), errors, errorhandler=errorhandler)
             return x.decode('utf-8') + y
 
         assert self.ll_to_string(self.interpret(f, [1])) == f(1)

From pypy.commits at gmail.com  Sat Feb 18 11:03:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Feb 2017 08:03:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: translation fixes
Message-ID: <58a8703e.c3d1190a.4dcd1.e4a0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90181:6a6b47616053
Date: 2017-02-18 17:02 +0100
http://bitbucket.org/pypy/pypy/changeset/6a6b47616053/

Log:	translation fixes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1553,6 +1553,13 @@
                 return w_obj.buffer_w(self, self.BUF_SIMPLE)
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes-like object", w_obj)
+        elif code == 'y#':
+            if self.isinstance_w(w_obj, self.w_bytes):
+                return w_obj.bytes_w(self)
+            try:
+                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_str()
+            except BufferInterfaceNotFound:
+                self._getarg_error("bytes-like object", w_obj)
         else:
             assert False
 
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -154,7 +154,7 @@
     # If allow_surrogates=True, then revert to the Python 2 behavior
     # which never raises UnicodeEncodeError.  Surrogate pairs are then
     # allowed, either paired or lone.  A paired surrogate is considered
-    # like the non-BMP character it stands for.  See also unicode_utf8sp().
+    # like the non-BMP character it stands for.  See also *_utf8sp().
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
         errorhandler=encode_error_handler(space),
diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -13,8 +13,6 @@
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
-from pypy.interpreter.error import strerror as _strerror
-
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
     includes=[cwd.join('locale_codec.h')],
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -167,12 +167,12 @@
     if w_value:
         w_error = space.call_function(w_type,
                                       space.newint(errno),
-                                      space.newtext(msg),
+                                      space.newunicode(msg),
                                       w_value)
     else:
         w_error = space.call_function(w_type,
                                       space.newint(errno),
-                                      space.newtext(msg))
+                                      space.newunicode(msg))
     raise OperationError(w_type, w_error)
 
 @cpython_api([], rffi.INT_real, error=-1)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -111,7 +111,8 @@
     space.setattr(w_mod, space.newtext('__file__'), space.newtext(filename))
     space.setattr(w_mod, space.newtext('__doc__'), space.w_None)
     if pkgdir is not None:
-        space.setattr(w_mod, space.newtext('__path__'), space.newlist([w(pkgdir)]))
+        space.setattr(w_mod, space.newtext('__path__'),
+                      space.newlist([space.newtext(pkgdir)]))
     init_extra_module_attrs(space, w_mod)
 
 def add_module(space, w_name):
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -53,7 +53,7 @@
     def find(self, w_tofind, w_start=None, w_end=None):
         self.check_valid()
         space = self.space
-        tofind = space.getarg_w('y*', w_tofind)
+        tofind = space.getarg_w('y#', w_tofind)
         if w_start is None:
             start = self.mmap.pos
         else:
@@ -67,7 +67,7 @@
     def rfind(self, w_tofind, w_start=None, w_end=None):
         self.check_valid()
         space = self.space
-        tofind = space.getarg_w('y*', w_tofind)
+        tofind = space.getarg_w('y#', w_tofind)
         if w_start is None:
             start = self.mmap.pos
         else:
@@ -99,7 +99,7 @@
 
     def write(self, w_data):
         self.check_valid()
-        data = self.space.getarg_w('y*', w_data)
+        data = self.space.getarg_w('y#', w_data)
         self.check_writeable()
         try:
             self.mmap.write(data)
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -573,7 +573,7 @@
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
-                space.newtext(_get_error_msg()))
+                                 space.newunicode(_get_error_msg()))
         return pbuf
 
     tup_w = space.fixedview(w_tup)
@@ -738,7 +738,8 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_ValueError, space.newtext(_get_error_msg()))
+        raise OperationError(space.w_ValueError,
+                             space.newunicode(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def localtime(space, w_seconds=None):
@@ -755,7 +756,8 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_OSError, space.newtext(_get_error_msg()))
+        raise OperationError(space.w_OSError,
+                             space.newunicode(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def mktime(space, w_tup):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -160,7 +160,7 @@
             try:
                 unicode_x = x.decode('ascii')
             except UnicodeDecodeError:
-                unicode_x = self._wrap_string_old(x)
+                return self._wrap_string_old(x)
             return self.newunicode(unicode_x)
         if isinstance(x, unicode):
             return self.newunicode(x)
@@ -176,8 +176,17 @@
 
     def _wrap_string_old(self, x):
         # XXX should disappear soon
-        print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
-        return self.newtext(x)
+        print 'WARNING: space.wrap() called on a non-ascii byte string: %s' % (
+            self.text_w(self.repr(self.newbytes(x))),)
+        lst = []
+        for ch in x:
+            ch = ord(ch)
+            if ch > 127:
+                lst.append(u'\ufffd')
+            else:
+                lst.append(unichr(ch))
+        unicode_x = u''.join(lst)
+        return self.newunicode(unicode_x)
 
     @not_rpython # only for tests
     def _wrap_not_rpython(self, x):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -424,9 +424,6 @@
     def descr_rmod(self, space, w_values):
         return mod_format(space, w_values, self, fmt_type=FORMAT_UNICODE)
 
-    def descr_rmod(self, space, w_values):
-        return mod_format(space, w_values, self, do_unicode=True)
-
     def descr_translate(self, space, w_table):
         selfvalue = self._value
         w_sys = space.getbuiltinmodule('sys')
@@ -617,7 +614,6 @@
         encoding = getdefaultencoding(space)
     if errors is None or errors == 'strict':
         if encoding == 'ascii':
-            # XXX error handling
             s = space.charbuf_w(w_obj)
             try:
                 u = fast_str_decode_ascii(s)

From pypy.commits at gmail.com  Sat Feb 18 11:42:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Feb 2017 08:42:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test_thread
Message-ID: <58a8798c.1a4b2e0a.b0c4d.ec2b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90182:5eff971973e1
Date: 2017-02-18 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/5eff971973e1/

Log:	fix test_thread

diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -4,7 +4,10 @@
 class TestThread(BaseTestPyPyC):
     def test_simple(self):
         def main(n):
-            import thread
+            try:
+                import _thread as thread
+            except ImportError:
+                import thread
             def f():
                 i = 0
                 while i < n:
@@ -29,7 +32,10 @@
 
     def test_tls(self):
         def main(n):
-            import thread
+            try:
+                import _thread as thread
+            except ImportError:
+                import thread
             local = thread._local()
             local.x = 1
             i = 0
@@ -64,10 +70,13 @@
         guard_true(i56, descr=...)
         p57 = force_token()
         setfield_gc(p0, p57, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token 8>)
-        i58 = call_release_gil_i(0, _, i37, 1, descr=<Calli 4 ii EF=7>)
+        i58 = call_may_force_i(ConstClass(acquire_timed), p31, -1, descr=<Calli . ri EF=7>)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
+        i99 = int_eq(i58, 1)
+        guard_true(i99, descr=...)
         i58 = int_sub(i44, 1)
+        p98 = getfield_gc_r(p97, descr=.*inst_sys_exc_operror.*)
         i59 = call_i(ConstClass(RPyThreadReleaseLock), i37, descr=<Calli . i EF=2>)
         i60 = int_is_true(i59)
         guard_false(i60, descr=...)

From pypy.commits at gmail.com  Sat Feb 18 11:50:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 18 Feb 2017 08:50:09 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-threads-gil: Move all GIL handling for
 PyGILState_* inside the functions
Message-ID: <58a87b41.010e2e0a.f948d.e9d0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-threads-gil
Changeset: r90183:8d233583a424
Date: 2017-02-18 16:47 +0000
http://bitbucket.org/pypy/pypy/changeset/8d233583a424/

Log:	Move all GIL handling for PyGILState_* inside the functions

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -266,9 +266,6 @@
         sig = pycode.cpython_code_signature(callable.func_code)
         assert sig.argnames[0] == 'space'
         self.argnames = sig.argnames[1:]
-        if gil == 'pygilstate_ensure':
-            assert self.argnames[-1] == 'previous_state'
-            del self.argnames[-1]
         assert len(self.argnames) == len(self.argtypes)
 
         self.gil = gil
@@ -851,15 +848,11 @@
         pypy_debug_catch_fatal_exception()
         assert False
 
-def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid):
+def _restore_gil_state(gil_release, _gil_auto, tid):
     from rpython.rlib import rgil
     # see "Handling of the GIL" above
     assert cpyext_glob_tid_ptr[0] == 0
-    if pygilstate_release:
-        from pypy.module.cpyext import pystate
-        unlock = (gilstate == pystate.PyGILState_UNLOCKED)
-    else:
-        unlock = gil_release or _gil_auto
+    unlock = gil_release or _gil_auto
     if unlock:
         rgil.release()
     else:
@@ -875,11 +868,9 @@
                                          # have the GIL, and acquire/release it
     gil_acquire = (gil == "acquire" or gil == "around")
     gil_release = (gil == "release" or gil == "around")
-    pygilstate_ensure = (gil == "pygilstate_ensure")
-    pygilstate_release = (gil == "pygilstate_release")
-    assert (gil is None or gil_acquire or gil_release
-            or pygilstate_ensure or pygilstate_release)
-    expected_nb_args = len(argtypesw) + pygilstate_ensure
+    gil_ignore = (gil == "ignore")
+    assert (gil is None or gil_acquire or gil_release or gil_ignore)
+    expected_nb_args = len(argtypesw)
 
     if isinstance(restype, lltype.Ptr) and error_value == 0:
         error_value = lltype.nullptr(restype.TO)
@@ -893,7 +884,6 @@
     def wrapper_second_level(callable, pname, *args):
         from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj
         from pypy.module.cpyext.pyobject import as_pyobj
-        from pypy.module.cpyext import pystate
         # we hope that malloc removal removes the newtuple() that is
         # inserted exactly here by the varargs specializer
 
@@ -905,21 +895,12 @@
                 deadlock_error(pname)
             rgil.acquire()
             assert cpyext_glob_tid_ptr[0] == 0
-        elif pygilstate_ensure:
-            if cpyext_glob_tid_ptr[0] == tid:
-                cpyext_glob_tid_ptr[0] = 0
-                args += (pystate.PyGILState_LOCKED,)
-            else:
-                rgil.acquire()
-                args += (pystate.PyGILState_UNLOCKED,)
+        elif gil_ignore:
+            pass
         else:
             if cpyext_glob_tid_ptr[0] != tid:
                 no_gil_error(pname)
             cpyext_glob_tid_ptr[0] = 0
-        if pygilstate_release:
-            gilstate = rffi.cast(lltype.Signed, args[-1])
-        else:
-            gilstate = pystate.PyGILState_IGNORE
 
         rffi.stackcounter.stacks_counter += 1
         llop.gc_stack_bottom(lltype.Void)   # marker for trackgcroot.py
@@ -943,8 +924,6 @@
                 else:
                     arg_conv = arg
                 boxed_args += (arg_conv, )
-            if pygilstate_ensure:
-                boxed_args += (args[-1], )
             try:
                 result = callable(space, *boxed_args)
                 if not we_are_translated() and DEBUG_WRAPPER:
@@ -989,14 +968,16 @@
 
         except Exception as e:
             unexpected_exception(pname, e, tb)
-            _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
+            if not gil_ignore:
+                _restore_gil_state(gil_release, _gil_auto, tid)
             state.check_and_raise_exception(always=True)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype
         rffi.stackcounter.stacks_counter -= 1
 
-        _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
+        if not gil_ignore:
+            _restore_gil_state(gil_release, _gil_auto, tid)
         return retval
 
     wrapper_second_level._dont_inline_ = True
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -1,10 +1,11 @@
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
+    cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct, cpyext_glob_tid_ptr)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref
 from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rthread
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import rgil
 
 PyInterpreterStateStruct = lltype.ForwardReference()
 PyInterpreterState = lltype.Ptr(PyInterpreterStateStruct)
@@ -232,11 +233,15 @@
     space.threadlocals.get_ec = get_possibly_deleted_ec
 
 
- at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL, gil="pygilstate_ensure")
-def PyGILState_Ensure(space, previous_state):
-    # The argument 'previous_state' is not part of the API; it is inserted
-    # by make_wrapper() and contains PyGILState_LOCKED/UNLOCKED based on
-    # the previous GIL state.
+ at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL, gil="ignore")
+def PyGILState_Ensure(space):
+    tid = rthread.get_or_make_ident()
+    if cpyext_glob_tid_ptr[0] == tid:
+        cpyext_glob_tid_ptr[0] = 0
+        previous_state = PyGILState_LOCKED
+    else:
+        rgil.acquire()
+        previous_state = PyGILState_UNLOCKED
     must_leave = space.threadlocals.try_enter_thread(space)
     ec = space.getexecutioncontext()
     if not must_leave:
@@ -254,10 +259,14 @@
         if not we_are_translated():
             _workaround_cpython_untranslated(space)
     #
+    cpyext_glob_tid_ptr[0] = tid
     return rffi.cast(PyGILState_STATE, previous_state)
 
- at cpython_api([PyGILState_STATE], lltype.Void, gil="pygilstate_release")
+ at cpython_api([PyGILState_STATE], lltype.Void, gil="ignore")
 def PyGILState_Release(space, oldstate):
+    tid = rthread.get_or_make_ident()
+    assert cpyext_glob_tid_ptr[0] == tid
+    cpyext_glob_tid_ptr[0] = 0
     oldstate = rffi.cast(lltype.Signed, oldstate)
     ec = space.getexecutioncontext()
     if ec.cpyext_gilstate_counter_noleave > 0:
@@ -266,6 +275,10 @@
         assert ec.cpyext_gilstate_counter_noleave == 0
         assert oldstate == PyGILState_UNLOCKED
         space.threadlocals.leave_thread(space)
+    if oldstate == PyGILState_UNLOCKED:
+        rgil.release()
+    else:
+        cpyext_glob_tid_ptr[0] = tid
 
 @cpython_api([], PyInterpreterState, error=CANNOT_FAIL)
 def PyInterpreterState_Head(space):

From pypy.commits at gmail.com  Sat Feb 18 15:36:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 18 Feb 2017 12:36:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: ok there is no point: the '_locale'
 module no longer hacks at the 'string' module in py3
Message-ID: <58a8b065.c59c190a.fbab7.f92e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90184:8c20ae4ead12
Date: 2017-02-18 21:36 +0100
http://bitbucket.org/pypy/pypy/changeset/8c20ae4ead12/

Log:	ok there is no point: the '_locale' module no longer hacks at the
	'string' module in py3

diff --git a/pypy/module/_locale/test/test_locale.py b/pypy/module/_locale/test/test_locale.py
--- a/pypy/module/_locale/test/test_locale.py
+++ b/pypy/module/_locale/test/test_locale.py
@@ -106,24 +106,6 @@
         assert _locale.setlocale(_locale.LC_ALL, None)
         assert _locale.setlocale(_locale.LC_ALL)
 
-    def test_string_ulcase(self):
-        if not hasattr(self, 'language_utf8'):
-            skip("No utf8 locale on this platform")
-        import _locale, string
-
-        lcase = "abcdefghijklmnopqrstuvwxyz"
-        ucase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-
-        _locale.setlocale(_locale.LC_ALL, self.language_utf8)
-        assert string.ascii_lowercase == lcase
-        assert string.ascii_uppercase == ucase
-
-        _locale.setlocale(_locale.LC_ALL, self.language_en)
-
-        # the asserts below are just plain wrong
-        #    assert string.lowercase != lcase
-        #    assert string.uppercase != ucase
-
     def test_localeconv(self):
         import _locale
 

From pypy.commits at gmail.com  Sat Feb 18 20:31:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 18 Feb 2017 17:31:58 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Add myself
Message-ID: <58a8f58e.12582e0a.ce423.ffb6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: extradoc
Changeset: r5782:c6e511be2c8f
Date: 2017-02-19 01:31 +0000
http://bitbucket.org/pypy/extradoc/changeset/c6e511be2c8f/

Log:	Add myself

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -14,6 +14,7 @@
 Remi Meier           (?)27.02/04.03 Ermina
 Manuel Jacob         25.02/04.03    Ermina
 Robert Zaremba       25.02/01.03    Ermina
+Ronan Lamy           25.02/04.03    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Sun Feb 19 00:03:58 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sat, 18 Feb 2017 21:03:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: 2to3
Message-ID: <58a9273e.010e2e0a.f948d.ffbd@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r90185:1f0462138b76
Date: 2017-02-19 06:02 +0100
http://bitbucket.org/pypy/pypy/changeset/1f0462138b76/

Log:	2to3

diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
--- a/pypy/module/__pypy__/test/test_magic.py
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -50,7 +50,7 @@
         assert decode_long(b'\xff\x7f', 'little', False) == 32767
         assert decode_long(b'\x00\x80', 'little', False) == 32768
         assert decode_long(b'\x00\x80', 'little', True) == -32768
-        raises(ValueError, decode_long, '', 'foo')
+        raises(ValueError, decode_long, b'', 'foo')
 
     def test_promote(self):
         from __pypy__ import _promote

From pypy.commits at gmail.com  Sun Feb 19 03:55:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 00:55:47 -0800 (PST)
Subject: [pypy-commit] pypy default: fix tests
Message-ID: <58a95d93.152c190a.9c982.089b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90186:56d384d17f52
Date: 2017-02-19 09:55 +0100
http://bitbucket.org/pypy/pypy/changeset/56d384d17f52/

Log:	fix tests

diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -492,7 +492,7 @@
 
 class TestPosixAscii(BasePosixUnicodeOrAscii):
     def _get_filename(self):
-        return str(udir.join('test_open_ascii'))
+        return unicode(udir.join('test_open_ascii'))
 
     @rposix_requires('openat')
     def test_openat(self):

From pypy.commits at gmail.com  Sun Feb 19 04:02:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:02:32 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <58a95f28.43502e0a.b5243.0780@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90187:9638991ff70e
Date: 2017-02-19 10:01 +0100
http://bitbucket.org/pypy/pypy/changeset/9638991ff70e/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -142,3 +142,11 @@
 it is the standard ``OrderedDict.move_to_end()`` method, but the
 behavior is also available on Python 2.x or for the ``dict`` type by
 calling ``__pypy__.move_to_end(dict, key, last=True)``.
+
+.. branch: space-newtext
+
+Internal refactoring of ``space.wrap()``, which is now replaced with
+explicitly-typed methods.  Notably, there are now ``space.newbytes()``
+and ``space.newtext()``: these two methods are identical on PyPy 2.7 but
+not on PyPy 3.x.  The latter is used to get an app-level unicode string
+by decoding the RPython string, assumed to be utf-8.

From pypy.commits at gmail.com  Sun Feb 19 04:06:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:06:08 -0800 (PST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <58a96000.8c092e0a.cd48d.083f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90188:c6bc044c5f15
Date: 2017-02-19 10:04 +0100
http://bitbucket.org/pypy/pypy/changeset/c6bc044c5f15/

Log:	fix

diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -174,7 +174,7 @@
                     m1, s12, m2, s23, m3, w_x)
 
 
-    @unwrap_spec(module_name='text', _version=int, _types='text')
+    @unwrap_spec(module_name='text', _version=int, _types='bytes')
     def descr_init(self, module_name='?', _version=-1, _types='',
                    w__globals=None, w__struct_unions=None, w__enums=None,
                    w__typenames=None, w__includes=None):

From pypy.commits at gmail.com  Sun Feb 19 04:06:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:06:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a96003.52152e0a.d48ff.0ace@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90189:50d92acc1afa
Date: 2017-02-19 10:04 +0100
http://bitbucket.org/pypy/pypy/changeset/50d92acc1afa/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -151,3 +151,11 @@
 it is the standard ``OrderedDict.move_to_end()`` method, but the
 behavior is also available on Python 2.x or for the ``dict`` type by
 calling ``__pypy__.move_to_end(dict, key, last=True)``.
+
+.. branch: space-newtext
+
+Internal refactoring of ``space.wrap()``, which is now replaced with
+explicitly-typed methods.  Notably, there are now ``space.newbytes()``
+and ``space.newtext()``: these two methods are identical on PyPy 2.7 but
+not on PyPy 3.x.  The latter is used to get an app-level unicode string
+by decoding the RPython string, assumed to be utf-8.
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -174,7 +174,7 @@
                     m1, s12, m2, s23, m3, w_x)
 
 
-    @unwrap_spec(module_name='text', _version=int, _types='text')
+    @unwrap_spec(module_name='text', _version=int, _types='bytes')
     def descr_init(self, module_name='?', _version=-1, _types='',
                    w__globals=None, w__struct_unions=None, w__enums=None,
                    w__typenames=None, w__includes=None):
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -492,7 +492,7 @@
 
 class TestPosixAscii(BasePosixUnicodeOrAscii):
     def _get_filename(self):
-        return str(udir.join('test_open_ascii'))
+        return unicode(udir.join('test_open_ascii'))
 
     @rposix_requires('openat')
     def test_openat(self):

From pypy.commits at gmail.com  Sun Feb 19 04:23:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:23:09 -0800 (PST)
Subject: [pypy-commit] pypy default: fix, shown randomly in some tests
Message-ID: <58a963fd.ce18190a.d70c9.275c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90190:2372b8d38e72
Date: 2017-02-19 10:22 +0100
http://bitbucket.org/pypy/pypy/changeset/2372b8d38e72/

Log:	fix, shown randomly in some tests

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -139,12 +139,12 @@
     # NB. a bit messy because rtyper/rstr.py also calls the same
     # function.  Make sure we annotate for the args it passes, too
     if NonConstant(False):
-        s = '?????'
-        size = 12345
-        errors = 'strict'
-        final = True
+        s = NonConstant('?????')
+        size = NonConstant(12345)
+        errors = NonConstant('strict')
+        final = NonConstant(True)
         errorhandler = default_unicode_error_decode
-        allow_surrogates = False
+        allow_surrogates = NonConstant(False)
     return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
@@ -356,11 +356,11 @@
     # NB. a bit messy because rtyper/rstr.py also calls the same
     # function.  Make sure we annotate for the args it passes, too
     if NonConstant(False):
-        s = u'?????'
-        size = 12345
-        errors = 'strict'
+        s = NonConstant(u'?????')
+        size = NonConstant(12345)
+        errors = NonConstant('strict')
         errorhandler = default_unicode_error_encode
-        allow_surrogates = False
+        allow_surrogates = NonConstant(False)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
 

From pypy.commits at gmail.com  Sun Feb 19 04:23:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:23:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a963ff.0d212e0a.6fb29.1293@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90191:072710fa60a3
Date: 2017-02-19 10:22 +0100
http://bitbucket.org/pypy/pypy/changeset/072710fa60a3/

Log:	hg merge default

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -139,12 +139,12 @@
     # NB. a bit messy because rtyper/rstr.py also calls the same
     # function.  Make sure we annotate for the args it passes, too
     if NonConstant(False):
-        s = '?????'
-        size = 12345
-        errors = 'strict'
-        final = True
+        s = NonConstant('?????')
+        size = NonConstant(12345)
+        errors = NonConstant('strict')
+        final = NonConstant(True)
         errorhandler = default_unicode_error_decode
-        allow_surrogates = False
+        allow_surrogates = NonConstant(False)
     return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
@@ -356,11 +356,11 @@
     # NB. a bit messy because rtyper/rstr.py also calls the same
     # function.  Make sure we annotate for the args it passes, too
     if NonConstant(False):
-        s = u'?????'
-        size = 12345
-        errors = 'strict'
+        s = NonConstant(u'?????')
+        size = NonConstant(12345)
+        errors = NonConstant('strict')
         errorhandler = default_unicode_error_encode
-        allow_surrogates = False
+        allow_surrogates = NonConstant(False)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
 

From pypy.commits at gmail.com  Sun Feb 19 04:26:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:26:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: typo
Message-ID: <58a964be.18532e0a.7645f.057f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90192:56a8ffad3837
Date: 2017-02-19 10:23 +0100
http://bitbucket.org/pypy/pypy/changeset/56a8ffad3837/

Log:	typo

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1507,7 +1507,7 @@
         if self.is_none(w_obj):
             e = oefmt(self.w_TypeError, "a %s is required, not None", expected)
         else:
-            e = oefmt(self.w_TypeError, "a %s is requried, not %T", expected, w_obj)
+            e = oefmt(self.w_TypeError, "a %s is required, not %T", expected, w_obj)
         raise e
 
     @specialize.arg(1)

From pypy.commits at gmail.com  Sun Feb 19 04:26:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:26:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test (two errors on the same call,
 and pypy happens to report the other one now)
Message-ID: <58a964c0.41152e0a.dc37.07cb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90193:a4715ec68be8
Date: 2017-02-19 10:25 +0100
http://bitbucket.org/pypy/pypy/changeset/a4715ec68be8/

Log:	fix test (two errors on the same call, and pypy happens to report
	the other one now)

diff --git a/lib-python/3/test/test_struct.py b/lib-python/3/test/test_struct.py
--- a/lib-python/3/test/test_struct.py
+++ b/lib-python/3/test/test_struct.py
@@ -542,7 +542,7 @@
 
         # format lists containing only count spec should result in an error
         self.assertRaises(struct.error, struct.pack, '12345')
-        self.assertRaises(struct.error, struct.unpack, '12345', '')
+        self.assertRaises(struct.error, struct.unpack, '12345', b'')
         self.assertRaises(struct.error, struct.pack_into, '12345', store, 0)
         self.assertRaises(struct.error, struct.unpack_from, '12345', store, 0)
 

From pypy.commits at gmail.com  Sun Feb 19 04:34:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 01:34:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <58a96691.0b8d190a.5cfd2.0e52@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90194:6d2b0db1c9cd
Date: 2017-02-19 10:33 +0100
http://bitbucket.org/pypy/pypy/changeset/6d2b0db1c9cd/

Log:	fix

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -457,7 +457,7 @@
             if do_unicode:
                 value = self.space.unicode_w(w_value)
             else:
-                value = self.space.bytes_w(w_value)
+                value = self.space.text_w(w_value)
             self.std_wp(value)
 
         def fmt_c(self, w_value):
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -408,6 +408,7 @@
         assert b"<%a>" % b"\xe9" == b"<b'\\xe9'>"
         assert b"<%a>" % "foo" == b"<'foo'>"
         assert b"<%a>" % "\u1234" == b"<'\\u1234'>"
+        assert b"<%a>" % 3.14 == b"<3.14>"
 
     def test_r_compat_bytes(self):
         assert b"<%r>" % b"test" == b"<b'test'>"

From pypy.commits at gmail.com  Sun Feb 19 06:41:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 03:41:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix merge issues: before the merge,
 the code (not the tests) was
Message-ID: <58a9845d.97002e0a.4ac70.176d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90195:0f157eedb3b2
Date: 2017-02-19 12:36 +0100
http://bitbucket.org/pypy/pypy/changeset/0f157eedb3b2/

Log:	fix merge issues: before the merge, the code (not the tests) was
	identical in pypy2 and pypy3, so just make it identical again

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -771,6 +771,7 @@
     for c_type, names, c_tc in type_info:
         class BasicConverter(ffitypes.typeid(c_type), IntTypeConverterMixin, TypeConverter):
             _immutable_ = True
+            typecode = c_tc
             def __init__(self, space, default):
                 self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -2,6 +2,7 @@
 
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat
+from rpython.rlib.rbigint import rbigint
 
 from pypy.module._cffi_backend import newtype
 
@@ -41,7 +42,6 @@
         self.c_size_t    = nt.new_primitive_type(space, 'size_t')
         self.c_ptrdiff_t = nt.new_primitive_type(space, 'ptrdiff_t')
 
-
 class BoolTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
@@ -49,6 +49,9 @@
     c_type      = rffi.UCHAR
     c_ptrtype   = rffi.UCHARP
 
+    def _wrap_object(self, space, obj):
+        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
+
     def _unwrap_object(self, space, w_obj):
         arg = space.c_int_w(w_obj)
         if arg != False and arg != True:
@@ -56,9 +59,6 @@
                         "boolean value should be bool, or integer 1 or 0")
         return arg
 
-    def _wrap_object(self, space, obj):
-        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
-
     def cffi_type(self, space):
         state = space.fromcache(State)
         return state.c_bool
@@ -93,44 +93,44 @@
         state = space.fromcache(State)
         return state.c_char
 
-class ShortTypeMixin(object):
+class BaseIntTypeMixin(object):
+    _mixin_     = True
+
+    def _wrap_object(self, space, obj):
+        return space.newint(rffi.cast(rffi.INT, obj))
+
+    def _unwrap_object(self, space, w_obj):
+        return rffi.cast(self.c_type, space.c_int_w(w_obj))
+
+class ShortTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
     c_type      = rffi.SHORT
     c_ptrtype   = rffi.SHORTP
 
-    def _unwrap_object(self, space, w_obj):
-        return rffi.cast(rffi.SHORT, space.int_w(w_obj))
-
+class UShortTypeMixin(BaseIntTypeMixin):
     def cffi_type(self, space):
         state = space.fromcache(State)
         return state.c_short
 
-class UShortTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
     c_type      = rffi.USHORT
     c_ptrtype   = rffi.USHORTP
 
-    def _unwrap_object(self, space, w_obj):
-        return rffi.cast(self.c_type, space.int_w(w_obj))
-
+class IntTypeMixin(BaseIntTypeMixin):
     def cffi_type(self, space):
         state = space.fromcache(State)
         return state.c_ushort
 
-class IntTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['c_type', 'c_ptrtype']
 
     c_type      = rffi.INT
     c_ptrtype   = rffi.INTP
 
-    def _unwrap_object(self, space, w_obj):
-        return rffi.cast(self.c_type, space.c_int_w(w_obj))
-
     def cffi_type(self, space):
         state = space.fromcache(State)
         return state.c_int

From pypy.commits at gmail.com  Sun Feb 19 07:03:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 04:03:22 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in
 gmarkall/pypy/fix_bool_restype (pull request #516)
Message-ID: <58a9898a.52152e0a.d48ff.11bc@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r90197:6b02806984f7
Date: 2017-02-19 12:02 +0000
http://bitbucket.org/pypy/pypy/changeset/6b02806984f7/

Log:	Merged in gmarkall/pypy/fix_bool_restype (pull request #516)

	Fix issue 2475: bool restypes not working

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -604,7 +604,8 @@
         """
         # hack for performance: if restype is a "simple" primitive type, don't
         # allocate the buffer because it's going to be thrown away immediately
-        if self._is_primitive(restype) and not restype._is_pointer_like():
+        if (self._is_primitive(restype) and restype._type_ != '?'
+            and not restype._is_pointer_like()):
             return result
         #
         shape = restype._ffishape_
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -99,6 +99,15 @@
         result = f(0, 0, 0, 0, 0, 0)
         assert result == '\x00'
 
+    def test_boolresult(self):
+        f = dll._testfunc_i_bhilfd
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_bool
+        false_result = f(0, 0, 0, 0, 0, 0)
+        assert false_result is False
+        true_result = f(1, 0, 0, 0, 0, 0)
+        assert true_result is True
+
     def test_voidresult(self):
         f = dll._testfunc_v
         f.restype = None

From pypy.commits at gmail.com  Sun Feb 19 07:03:30 2017
From: pypy.commits at gmail.com (Graham Markall)
Date: Sun, 19 Feb 2017 04:03:30 -0800 (PST)
Subject: [pypy-commit] pypy fix_bool_restype: Fix issue 2475: bool restypes
 not working
Message-ID: <58a98992.0b8d190a.5cfd2.1482@mx.google.com>

Author: Graham Markall <graham.markall at embecosm.com>
Branch: fix_bool_restype
Changeset: r90196:9bb716c6f836
Date: 2017-02-15 18:44 +0000
http://bitbucket.org/pypy/pypy/changeset/9bb716c6f836/

Log:	Fix issue 2475: bool restypes not working

	This commit adds a check to the performance hack in
	CFuncPtr._wrap_result that doesn't use the hack when the return type
	is bool, because a new bool object needs to be created.

	A test of a C function with a bool restype is added.

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -604,7 +604,8 @@
         """
         # hack for performance: if restype is a "simple" primitive type, don't
         # allocate the buffer because it's going to be thrown away immediately
-        if self._is_primitive(restype) and not restype._is_pointer_like():
+        if (self._is_primitive(restype) and restype._type_ != '?'
+            and not restype._is_pointer_like()):
             return result
         #
         shape = restype._ffishape_
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -99,6 +99,15 @@
         result = f(0, 0, 0, 0, 0, 0)
         assert result == '\x00'
 
+    def test_boolresult(self):
+        f = dll._testfunc_i_bhilfd
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_bool
+        false_result = f(0, 0, 0, 0, 0, 0)
+        assert false_result is False
+        true_result = f(1, 0, 0, 0, 0, 0)
+        assert true_result is True
+
     def test_voidresult(self):
         f = dll._testfunc_v
         f.restype = None

From pypy.commits at gmail.com  Sun Feb 19 08:37:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 05:37:50 -0800 (PST)
Subject: [pypy-commit] cffi default: issue #255: comparing primitive cdatas
Message-ID: <58a99fae.049f190a.5a9c1.b6c7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2890:91fcbd69bce2
Date: 2017-02-19 14:37 +0100
http://bitbucket.org/cffi/cffi/changeset/91fcbd69bce2/

Log:	issue #255: comparing primitive cdatas

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2031,47 +2031,97 @@
 
 static PyObject *cdata_richcompare(PyObject *v, PyObject *w, int op)
 {
-    int res;
+    int v_is_ptr, w_is_ptr;
     PyObject *pyres;
-    char *v_cdata, *w_cdata;
 
     assert(CData_Check(v));
-    if (!CData_Check(w)) {
+
+    /* Comparisons involving a primitive cdata work differently than
+     * comparisons involving a struct/array/pointer.
+     *
+     * If v or w is a struct/array/pointer, then the other must be too
+     * (otherwise we return NotImplemented and leave the case to
+     * Python).  If both are, then we compare the addresses.
+     *
+     * If v and/or w is a primitive cdata, then we convert the cdata(s)
+     * to regular Python objects and redo the comparison there.
+     */
+
+    v_is_ptr = !(((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY);
+    w_is_ptr = CData_Check(w) &&
+                  !(((CDataObject *)w)->c_type->ct_flags & CT_PRIMITIVE_ANY);
+
+    if (v_is_ptr && w_is_ptr) {
+        int res;
+        char *v_cdata = ((CDataObject *)v)->c_data;
+        char *w_cdata = ((CDataObject *)w)->c_data;
+
+        switch (op) {
+        case Py_EQ: res = (v_cdata == w_cdata); break;
+        case Py_NE: res = (v_cdata != w_cdata); break;
+        case Py_LT: res = (v_cdata <  w_cdata); break;
+        case Py_LE: res = (v_cdata <= w_cdata); break;
+        case Py_GT: res = (v_cdata >  w_cdata); break;
+        case Py_GE: res = (v_cdata >= w_cdata); break;
+        default: res = -1;
+        }
+        pyres = res ? Py_True : Py_False;
+    }
+    else if (v_is_ptr || w_is_ptr) {
         pyres = Py_NotImplemented;
-        goto done;
-    }
-
-    if ((op != Py_EQ && op != Py_NE) &&
-        ((((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) ||
-         (((CDataObject *)w)->c_type->ct_flags & CT_PRIMITIVE_ANY)))
-        goto Error;
-
-    v_cdata = ((CDataObject *)v)->c_data;
-    w_cdata = ((CDataObject *)w)->c_data;
-
-    switch (op) {
-    case Py_EQ: res = (v_cdata == w_cdata); break;
-    case Py_NE: res = (v_cdata != w_cdata); break;
-    case Py_LT: res = (v_cdata <  w_cdata); break;
-    case Py_LE: res = (v_cdata <= w_cdata); break;
-    case Py_GT: res = (v_cdata >  w_cdata); break;
-    case Py_GE: res = (v_cdata >= w_cdata); break;
-    default: res = -1;
-    }
-    pyres = res ? Py_True : Py_False;
- done:
+    }
+    else {
+        PyObject *aa[2];
+        int i;
+
+        aa[0] = v; Py_INCREF(v);
+        aa[1] = w; Py_INCREF(w);
+        pyres = NULL;
+
+        for (i = 0; i < 2; i++) {
+            v = aa[i];
+            if (!CData_Check(v))
+                continue;
+            w = convert_to_object(((CDataObject *)v)->c_data,
+                                  ((CDataObject *)v)->c_type);
+            if (w == NULL)
+                goto error;
+            if (CData_Check(w)) {
+                Py_DECREF(w);
+                PyErr_Format(PyExc_NotImplementedError,
+                             "cannot use <cdata '%s'> in a comparison",
+                             ((CDataObject *)v)->c_type->ct_name);
+                goto error;
+            }
+            aa[i] = w;
+            Py_DECREF(v);
+        }
+        pyres = PyObject_RichCompare(aa[0], aa[1], op);
+     error:
+        Py_DECREF(aa[1]);
+        Py_DECREF(aa[0]);
+        return pyres;
+    }
+
     Py_INCREF(pyres);
     return pyres;
-
- Error:
-    PyErr_SetString(PyExc_TypeError,
-                    "cannot do comparison on a primitive cdata");
-    return NULL;
-}
-
-static long cdata_hash(CDataObject *cd)
-{
-    return _Py_HashPointer(cd->c_data);
+}
+
+static long cdata_hash(CDataObject *v)
+{
+    if (((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) {
+        PyObject *vv = convert_to_object(((CDataObject *)v)->c_data,
+                                         ((CDataObject *)v)->c_type);
+        if (vv == NULL)
+            return -1;
+        if (!CData_Check(vv)) {
+            long hash = PyObject_Hash(vv);
+            Py_DECREF(vv);
+            return hash;
+        }
+        Py_DECREF(vv);
+    }
+    return _Py_HashPointer(v->c_data);
 }
 
 static Py_ssize_t
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -27,6 +27,7 @@
                                        .replace(r'\\U', r'\U'))
     u = U()
     str2bytes = str
+    strict_compare = False
 else:
     type_or_class = "class"
     long = int
@@ -38,6 +39,7 @@
     bitem2bchr = bytechr
     u = ""
     str2bytes = lambda s: bytes(s, "ascii")
+    strict_compare = True
 
 def size_of_int():
     BInt = new_primitive_type("int")
@@ -106,11 +108,11 @@
     x = cast(p, -66 + (1<<199)*256)
     assert repr(x) == "<cdata 'signed char' -66>"
     assert int(x) == -66
-    assert (x == cast(p, -66)) is False
-    assert (x != cast(p, -66)) is True
+    assert (x == cast(p, -66)) is True
+    assert (x != cast(p, -66)) is False
     q = new_primitive_type("short")
-    assert (x == cast(q, -66)) is False
-    assert (x != cast(q, -66)) is True
+    assert (x == cast(q, -66)) is True
+    assert (x != cast(q, -66)) is False
 
 def test_sizeof_type():
     py.test.raises(TypeError, sizeof, 42.5)
@@ -175,7 +177,7 @@
             assert float(cast(p, 1.1)) != 1.1     # rounding error
             assert float(cast(p, 1E200)) == INF   # limited range
 
-        assert cast(p, -1.1) != cast(p, -1.1)
+        assert cast(p, -1.1) == cast(p, -1.1)
         assert repr(float(cast(p, -0.0))) == '-0.0'
         assert float(cast(p, b'\x09')) == 9.0
         assert float(cast(p, u+'\x09')) == 9.0
@@ -219,7 +221,7 @@
     p = new_primitive_type("char")
     assert bool(cast(p, 'A')) is True
     assert bool(cast(p, '\x00')) is False    # since 1.7
-    assert cast(p, '\x00') != cast(p, -17*256)
+    assert cast(p, '\x00') == cast(p, -17*256)
     assert int(cast(p, 'A')) == 65
     assert long(cast(p, 'A')) == 65
     assert type(int(cast(p, 'A'))) is int
@@ -382,23 +384,6 @@
     # that it is already loaded too, so it should work
     assert x.load_function(BVoidP, 'sqrt')
 
-def test_hash_differences():
-    BChar = new_primitive_type("char")
-    BInt = new_primitive_type("int")
-    BFloat = new_primitive_type("float")
-    for i in range(1, 20):
-        x1 = cast(BChar, chr(i))
-        x2 = cast(BInt, i)
-        if hash(x1) != hash(x2):
-            break
-    else:
-        raise AssertionError("hashes are equal")
-    for i in range(1, 20):
-        if hash(cast(BFloat, i)) != hash(float(i)):
-            break
-    else:
-        raise AssertionError("hashes are equal")
-
 def test_no_len_on_nonarray():
     p = new_primitive_type("int")
     py.test.raises(TypeError, len, cast(p, 42))
@@ -2261,12 +2246,17 @@
     BVoidP = new_pointer_type(new_void_type())
     p = newp(BIntP, 123)
     q = cast(BInt, 124)
-    py.test.raises(TypeError, "p < q")
-    py.test.raises(TypeError, "p <= q")
     assert (p == q) is False
     assert (p != q) is True
-    py.test.raises(TypeError, "p > q")
-    py.test.raises(TypeError, "p >= q")
+    assert (q == p) is False
+    assert (q != p) is True
+    if strict_compare:
+        py.test.raises(TypeError, "p < q")
+        py.test.raises(TypeError, "p <= q")
+        py.test.raises(TypeError, "q < p")
+        py.test.raises(TypeError, "q <= p")
+        py.test.raises(TypeError, "p > q")
+        py.test.raises(TypeError, "p >= q")
     r = cast(BVoidP, p)
     assert (p <  r) is False
     assert (p <= r) is True
@@ -3840,3 +3830,86 @@
         assert len(w) == 2
     # check that the warnings are associated with lines in this file
     assert w[1].lineno == w[0].lineno + 4
+
+def test_primitive_comparison():
+    def assert_eq(a, b):
+        assert (a == b) is True
+        assert (b == a) is True
+        assert (a != b) is False
+        assert (b != a) is False
+        assert (a < b) is False
+        assert (a <= b) is True
+        assert (a > b) is False
+        assert (a >= b) is True
+        assert (b < a) is False
+        assert (b <= a) is True
+        assert (b > a) is False
+        assert (b >= a) is True
+        assert hash(a) == hash(b)
+    def assert_lt(a, b):
+        assert (a == b) is False
+        assert (b == a) is False
+        assert (a != b) is True
+        assert (b != a) is True
+        assert (a < b) is True
+        assert (a <= b) is True
+        assert (a > b) is False
+        assert (a >= b) is False
+        assert (b < a) is False
+        assert (b <= a) is False
+        assert (b > a) is True
+        assert (b >= a) is True
+        assert hash(a) != hash(b)    # (or at least, it is unlikely)
+    def assert_gt(a, b):
+        assert_lt(b, a)
+    def assert_ne(a, b):
+        assert (a == b) is False
+        assert (b == a) is False
+        assert (a != b) is True
+        assert (b != a) is True
+        if strict_compare:
+            py.test.raises(TypeError, "a < b")
+            py.test.raises(TypeError, "a <= b")
+            py.test.raises(TypeError, "a > b")
+            py.test.raises(TypeError, "a >= b")
+            py.test.raises(TypeError, "b < a")
+            py.test.raises(TypeError, "b <= a")
+            py.test.raises(TypeError, "b > a")
+            py.test.raises(TypeError, "b >= a")
+        elif a < b:
+            assert_lt(a, b)
+        else:
+            assert_lt(b, a)
+    assert_eq(5, 5)
+    assert_lt(3, 5)
+    assert_ne('5', 5)
+    #
+    t1 = new_primitive_type("char")
+    t2 = new_primitive_type("int")
+    t3 = new_primitive_type("unsigned char")
+    t4 = new_primitive_type("unsigned int")
+    t5 = new_primitive_type("float")
+    t6 = new_primitive_type("double")
+    assert_eq(cast(t1, 65), b'A')
+    assert_lt(cast(t1, 64), b'\x99')
+    assert_gt(cast(t1, 200), b'A')
+    assert_ne(cast(t1, 65), 65)
+    assert_eq(cast(t2, -25), -25)
+    assert_lt(cast(t2, -25), -24)
+    assert_gt(cast(t2, -25), -26)
+    assert_eq(cast(t3, 65), 65)
+    assert_ne(cast(t3, 65), b'A')
+    assert_ne(cast(t3, 65), cast(t1, 65))
+    assert_gt(cast(t4, -1), -1)
+    assert_gt(cast(t4, -1), cast(t2, -1))
+    assert_gt(cast(t4, -1), 99999)
+    assert_eq(cast(t4, -1), 256 ** size_of_int() - 1)
+    assert_eq(cast(t5, 3.0), 3)
+    assert_eq(cast(t5, 3.5), 3.5)
+    assert_lt(cast(t5, 3.3), 3.3)   # imperfect rounding
+    assert_eq(cast(t6, 3.3), 3.3)
+    assert_eq(cast(t5, 3.5), cast(t6, 3.5))
+    assert_lt(cast(t5, 3.1), cast(t6, 3.1))   # imperfect rounding
+    assert_eq(cast(t5, 7.0), cast(t3, 7))
+    assert_lt(cast(t5, 3.1), 3.101)
+    assert_gt(cast(t5, 3.1), 3)
diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -112,11 +112,20 @@
     def _make_cmp(name):
         cmpfunc = getattr(operator, name)
         def cmp(self, other):
-            if isinstance(other, CTypesData):
+            v_is_ptr = not isinstance(self, CTypesGenericPrimitive)
+            w_is_ptr = (isinstance(other, CTypesData) and
+                           not isinstance(other, CTypesGenericPrimitive))
+            if v_is_ptr and w_is_ptr:
                 return cmpfunc(self._convert_to_address(None),
                                other._convert_to_address(None))
+            elif v_is_ptr or w_is_ptr:
+                return NotImplemented
             else:
-                return NotImplemented
+                if isinstance(self, CTypesGenericPrimitive):
+                    self = self._value
+                if isinstance(other, CTypesGenericPrimitive):
+                    other = other._value
+                return cmpfunc(self, other)
         cmp.func_name = name
         return cmp
 
@@ -128,7 +137,7 @@
     __ge__ = _make_cmp('__ge__')
 
     def __hash__(self):
-        return hash(type(self)) ^ hash(self._convert_to_address(None))
+        return hash(self._convert_to_address(None))
 
     def _to_string(self, maxlen):
         raise TypeError("string(): %r" % (self,))
@@ -137,14 +146,8 @@
 class CTypesGenericPrimitive(CTypesData):
     __slots__ = []
 
-    def __eq__(self, other):
-        return self is other
-
-    def __ne__(self, other):
-        return self is not other
-
     def __hash__(self):
-        return object.__hash__(self)
+        return hash(self._value)
 
     def _get_own_repr(self):
         return repr(self._from_ctypes(self._value))
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -602,21 +602,21 @@
 |    C type     |   writing into         | reading from     |other operations|
 +===============+========================+==================+================+
 |   integers    | an integer or anything | a Python int or  | int(), bool()  |
-|   and enums   | on which int() works   | long, depending  | `(******)`     |
-|   `(*****)`   | (but not a float!).    | on the type      |                |
+|   and enums   | on which int() works   | long, depending  | `(******)`,    |
+|   `(*****)`   | (but not a float!).    | on the type      | ``<``          |
 |               | Must be within range.  | (ver. 1.10: or a |                |
 |               |                        | bool)            |                |
 +---------------+------------------------+------------------+----------------+
-|   ``char``    | a string of length 1   | a string of      | int(), bool()  |
-|               | or another <cdata char>| length 1         |                |
+|   ``char``    | a string of length 1   | a string of      | int(), bool(), |
+|               | or another <cdata char>| length 1         | ``<``          |
 +---------------+------------------------+------------------+----------------+
 |  ``wchar_t``  | a unicode of length 1  | a unicode of     |                |
-|               | (or maybe 2 if         | length 1         | int(), bool()  |
-|               | surrogates) or         | (or maybe 2 if   |                |
+|               | (or maybe 2 if         | length 1         | int(), bool(), |
+|               | surrogates) or         | (or maybe 2 if   | ``<``          |
 |               | another <cdata wchar_t>| surrogates)      |                |
 +---------------+------------------------+------------------+----------------+
 |  ``float``,   | a float or anything on | a Python float   | float(), int(),|
-|  ``double``   | which float() works    |                  | bool()         |
+|  ``double``   | which float() works    |                  | bool(), ``<``  |
 +---------------+------------------------+------------------+----------------+
 |``long double``| another <cdata> with   | a <cdata>, to    | float(), int(),|
 |               | a ``long double``, or  | avoid loosing    | bool()         |
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -50,6 +50,13 @@
   only in out-of-line mode.  This is useful for taking the address of
   global variables.
 
+* Issue #255: ``cdata`` objects of a primitive type (integers, floats,
+  char) are now compared and ordered by value.  For example, ``<cdata
+  'int' 42>`` compares equal to ``42`` and ``<cdata 'char' b'A'>``
+  compares equal to ``b'A'``.  Unlike C, ``<cdata 'int' -1>`` does not
+  compare equal to ``ffi.cast("unsigned int", -1)``: it compares
+  smaller, because ``-1 < 4294967295``.
+
 
 v1.9
 ====
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -54,7 +54,8 @@
         min = int(min)
         max = int(max)
         p = ffi.cast(c_decl, min)
-        assert p != min       # no __eq__(int)
+        assert p == min
+        assert hash(p) == hash(min)
         assert bool(p) is bool(min)
         assert int(p) == min
         p = ffi.cast(c_decl, max)
@@ -65,9 +66,9 @@
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
         q = ffi.cast(c_decl, long(min - 1))
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
-        assert q != p
+        assert q == p
         assert int(q) == int(p)
-        assert hash(q) != hash(p)   # unlikely
+        assert hash(q) == hash(p)
         c_decl_ptr = '%s *' % c_decl
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, min - 1)
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, max + 1)
@@ -882,9 +883,9 @@
         assert ffi.string(ffi.cast("enum bar", -2)) == "B1"
         assert ffi.string(ffi.cast("enum bar", -1)) == "CC1"
         assert ffi.string(ffi.cast("enum bar", 1)) == "E1"
-        assert ffi.cast("enum bar", -2) != ffi.cast("enum bar", -2)
-        assert ffi.cast("enum foo", 0) != ffi.cast("enum bar", 0)
-        assert ffi.cast("enum bar", 0) != ffi.cast("int", 0)
+        assert ffi.cast("enum bar", -2) == ffi.cast("enum bar", -2)
+        assert ffi.cast("enum foo", 0) == ffi.cast("enum bar", 0)
+        assert ffi.cast("enum bar", 0) == ffi.cast("int", 0)
         assert repr(ffi.cast("enum bar", -1)) == "<cdata 'enum bar' -1: CC1>"
         assert repr(ffi.cast("enum foo", -1)) == (  # enums are unsigned, if
             "<cdata 'enum foo' 4294967295>")        # they contain no neg value
@@ -1113,15 +1114,15 @@
         assert (q == None) is False
         assert (q != None) is True
 
-    def test_no_integer_comparison(self):
+    def test_integer_comparison(self):
         ffi = FFI(backend=self.Backend())
         x = ffi.cast("int", 123)
         y = ffi.cast("int", 456)
-        py.test.raises(TypeError, "x < y")
+        assert x < y
         #
         z = ffi.cast("double", 78.9)
-        py.test.raises(TypeError, "x < z")
-        py.test.raises(TypeError, "z < y")
+        assert x > z
+        assert y > z
 
     def test_ffi_buffer_ptr(self):
         ffi = FFI(backend=self.Backend())
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -137,7 +137,7 @@
         min = int(min)
         max = int(max)
         p = ffi.cast(c_decl, min)
-        assert p != min       # no __eq__(int)
+        assert p == min
         assert bool(p) is bool(min)
         assert int(p) == min
         p = ffi.cast(c_decl, max)
@@ -148,9 +148,9 @@
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
         q = ffi.cast(c_decl, long(min - 1))
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
-        assert q != p
+        assert q == p
         assert int(q) == int(p)
-        assert hash(q) != hash(p)   # unlikely
+        assert hash(q) == hash(p)
         c_decl_ptr = '%s *' % c_decl
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, min - 1)
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, max + 1)
@@ -896,9 +896,9 @@
         assert ffi.string(ffi.cast("enum bar", -2)) == "B1"
         assert ffi.string(ffi.cast("enum bar", -1)) == "CC1"
         assert ffi.string(ffi.cast("enum bar", 1)) == "E1"
-        assert ffi.cast("enum bar", -2) != ffi.cast("enum bar", -2)
-        assert ffi.cast("enum foq", 0) != ffi.cast("enum bar", 0)
-        assert ffi.cast("enum bar", 0) != ffi.cast("int", 0)
+        assert ffi.cast("enum bar", -2) == ffi.cast("enum bar", -2)
+        assert ffi.cast("enum foq", 0) == ffi.cast("enum bar", 0)
+        assert ffi.cast("enum bar", 0) == ffi.cast("int", 0)
         assert repr(ffi.cast("enum bar", -1)) == "<cdata 'enum bar' -1: CC1>"
         assert repr(ffi.cast("enum foq", -1)) == (  # enums are unsigned, if
             "<cdata 'enum foq' 4294967295>") or (   # they contain no neg value
@@ -1105,14 +1105,14 @@
         assert (q == None) is False
         assert (q != None) is True
 
-    def test_no_integer_comparison(self):
+    def test_integer_comparison(self):
         x = ffi.cast("int", 123)
         y = ffi.cast("int", 456)
-        py.test.raises(TypeError, "x < y")
+        assert x < y
         #
         z = ffi.cast("double", 78.9)
-        py.test.raises(TypeError, "x < z")
-        py.test.raises(TypeError, "z < y")
+        assert x > z
+        assert y > z
 
     def test_ffi_buffer_ptr(self):
         a = ffi.new("short *", 100)

From pypy.commits at gmail.com  Sun Feb 19 09:54:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 06:54:00 -0800 (PST)
Subject: [pypy-commit] pypy default: update to cffi/91fcbd69bce2: cffi issue
 #255
Message-ID: <58a9b188.13a6190a.1b698.195d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90198:87cd234e833d
Date: 2017-02-19 15:53 +0100
http://bitbucket.org/pypy/pypy/changeset/87cd234e833d/

Log:	update to cffi/91fcbd69bce2: cffi issue #255

diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -112,11 +112,20 @@
     def _make_cmp(name):
         cmpfunc = getattr(operator, name)
         def cmp(self, other):
-            if isinstance(other, CTypesData):
+            v_is_ptr = not isinstance(self, CTypesGenericPrimitive)
+            w_is_ptr = (isinstance(other, CTypesData) and
+                           not isinstance(other, CTypesGenericPrimitive))
+            if v_is_ptr and w_is_ptr:
                 return cmpfunc(self._convert_to_address(None),
                                other._convert_to_address(None))
+            elif v_is_ptr or w_is_ptr:
+                return NotImplemented
             else:
-                return NotImplemented
+                if isinstance(self, CTypesGenericPrimitive):
+                    self = self._value
+                if isinstance(other, CTypesGenericPrimitive):
+                    other = other._value
+                return cmpfunc(self, other)
         cmp.func_name = name
         return cmp
 
@@ -128,7 +137,7 @@
     __ge__ = _make_cmp('__ge__')
 
     def __hash__(self):
-        return hash(type(self)) ^ hash(self._convert_to_address(None))
+        return hash(self._convert_to_address(None))
 
     def _to_string(self, maxlen):
         raise TypeError("string(): %r" % (self,))
@@ -137,14 +146,8 @@
 class CTypesGenericPrimitive(CTypesData):
     __slots__ = []
 
-    def __eq__(self, other):
-        return self is other
-
-    def __ne__(self, other):
-        return self is not other
-
     def __hash__(self):
-        return object.__hash__(self)
+        return hash(self._value)
 
     def _get_own_repr(self):
         return repr(self._from_ctypes(self._value))
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -99,27 +99,39 @@
         raise oefmt(space.w_TypeError,
                     "cdata of type '%s' has no len()", self.ctype.name)
 
+    def _compare_mode(self, w_other):
+        from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
+        zero = rffi.cast(lltype.Unsigned, 0)
+        self_is_ptr = not isinstance(self.ctype, W_CTypePrimitive)
+        other_is_ptr = (isinstance(w_other, W_CData) and
+                           not isinstance(w_other.ctype, W_CTypePrimitive))
+        if other_is_ptr and self_is_ptr:
+            with self as ptr1, w_other as ptr2:
+                ptr1 = rffi.cast(lltype.Unsigned, ptr1)
+                ptr2 = rffi.cast(lltype.Unsigned, ptr2)
+            return (0, ptr1, ptr2, None, None)
+        elif other_is_ptr or self_is_ptr:
+            return (-1, zero, zero, None, None)
+        else:
+            w_ob1 = self.convert_to_object()
+            if isinstance(w_other, W_CData):
+                w_ob2 = w_other.convert_to_object()
+            else:
+                w_ob2 = w_other
+            return (1, zero, zero, w_ob1, w_ob2)
+
     def _make_comparison(name):
         op = getattr(operator, name)
-        requires_ordering = name not in ('eq', 'ne')
         #
         def _cmp(self, w_other):
-            from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
             space = self.space
-            if not isinstance(w_other, W_CData):
+            mode, adr1, adr2, w_ob1, w_ob2 = self._compare_mode(w_other)
+            if mode == 0:
+                return space.newbool(op(adr1, adr2))
+            elif mode == 1:
+                return getattr(space, name)(w_ob1, w_ob2)
+            else:
                 return space.w_NotImplemented
-
-            with self as ptr1, w_other as ptr2:
-                if requires_ordering:
-                    if (isinstance(self.ctype, W_CTypePrimitive) or
-                        isinstance(w_other.ctype, W_CTypePrimitive)):
-                        raise oefmt(space.w_TypeError,
-                                    "cannot do comparison on a primitive "
-                                    "cdata")
-                    ptr1 = rffi.cast(lltype.Unsigned, ptr1)
-                    ptr2 = rffi.cast(lltype.Unsigned, ptr2)
-                result = op(ptr1, ptr2)
-            return space.newbool(result)
         #
         return func_with_new_name(_cmp, name)
 
@@ -131,6 +143,11 @@
     ge = _make_comparison('ge')
 
     def hash(self):
+        from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
+        if isinstance(self.ctype, W_CTypePrimitive):
+            w_ob = self.convert_to_object()
+            if not isinstance(w_ob, W_CData):
+                return self.space.hash(w_ob)
         ptr = self.unsafe_escaping_ptr()
         h = rffi.cast(lltype.Signed, ptr)
         # To hash pointers in dictionaries.  Assumes that h shows some
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -16,6 +16,7 @@
                                        .replace(r'\\U', r'\U'))
     u = U()
     str2bytes = str
+    strict_compare = False
 else:
     type_or_class = "class"
     long = int
@@ -27,6 +28,7 @@
     bitem2bchr = bytechr
     u = ""
     str2bytes = lambda s: bytes(s, "ascii")
+    strict_compare = True
 
 def size_of_int():
     BInt = new_primitive_type("int")
@@ -95,11 +97,11 @@
     x = cast(p, -66 + (1<<199)*256)
     assert repr(x) == "<cdata 'signed char' -66>"
     assert int(x) == -66
-    assert (x == cast(p, -66)) is False
-    assert (x != cast(p, -66)) is True
+    assert (x == cast(p, -66)) is True
+    assert (x != cast(p, -66)) is False
     q = new_primitive_type("short")
-    assert (x == cast(q, -66)) is False
-    assert (x != cast(q, -66)) is True
+    assert (x == cast(q, -66)) is True
+    assert (x != cast(q, -66)) is False
 
 def test_sizeof_type():
     py.test.raises(TypeError, sizeof, 42.5)
@@ -164,7 +166,7 @@
             assert float(cast(p, 1.1)) != 1.1     # rounding error
             assert float(cast(p, 1E200)) == INF   # limited range
 
-        assert cast(p, -1.1) != cast(p, -1.1)
+        assert cast(p, -1.1) == cast(p, -1.1)
         assert repr(float(cast(p, -0.0))) == '-0.0'
         assert float(cast(p, b'\x09')) == 9.0
         assert float(cast(p, u+'\x09')) == 9.0
@@ -208,7 +210,7 @@
     p = new_primitive_type("char")
     assert bool(cast(p, 'A')) is True
     assert bool(cast(p, '\x00')) is False    # since 1.7
-    assert cast(p, '\x00') != cast(p, -17*256)
+    assert cast(p, '\x00') == cast(p, -17*256)
     assert int(cast(p, 'A')) == 65
     assert long(cast(p, 'A')) == 65
     assert type(int(cast(p, 'A'))) is int
@@ -371,23 +373,6 @@
     # that it is already loaded too, so it should work
     assert x.load_function(BVoidP, 'sqrt')
 
-def test_hash_differences():
-    BChar = new_primitive_type("char")
-    BInt = new_primitive_type("int")
-    BFloat = new_primitive_type("float")
-    for i in range(1, 20):
-        x1 = cast(BChar, chr(i))
-        x2 = cast(BInt, i)
-        if hash(x1) != hash(x2):
-            break
-    else:
-        raise AssertionError("hashes are equal")
-    for i in range(1, 20):
-        if hash(cast(BFloat, i)) != hash(float(i)):
-            break
-    else:
-        raise AssertionError("hashes are equal")
-
 def test_no_len_on_nonarray():
     p = new_primitive_type("int")
     py.test.raises(TypeError, len, cast(p, 42))
@@ -2250,12 +2235,17 @@
     BVoidP = new_pointer_type(new_void_type())
     p = newp(BIntP, 123)
     q = cast(BInt, 124)
-    py.test.raises(TypeError, "p < q")
-    py.test.raises(TypeError, "p <= q")
     assert (p == q) is False
     assert (p != q) is True
-    py.test.raises(TypeError, "p > q")
-    py.test.raises(TypeError, "p >= q")
+    assert (q == p) is False
+    assert (q != p) is True
+    if strict_compare:
+        py.test.raises(TypeError, "p < q")
+        py.test.raises(TypeError, "p <= q")
+        py.test.raises(TypeError, "q < p")
+        py.test.raises(TypeError, "q <= p")
+        py.test.raises(TypeError, "p > q")
+        py.test.raises(TypeError, "p >= q")
     r = cast(BVoidP, p)
     assert (p <  r) is False
     assert (p <= r) is True
@@ -3829,3 +3819,86 @@
         assert len(w) == 2
     # check that the warnings are associated with lines in this file
     assert w[1].lineno == w[0].lineno + 4
+
+def test_primitive_comparison():
+    def assert_eq(a, b):
+        assert (a == b) is True
+        assert (b == a) is True
+        assert (a != b) is False
+        assert (b != a) is False
+        assert (a < b) is False
+        assert (a <= b) is True
+        assert (a > b) is False
+        assert (a >= b) is True
+        assert (b < a) is False
+        assert (b <= a) is True
+        assert (b > a) is False
+        assert (b >= a) is True
+        assert hash(a) == hash(b)
+    def assert_lt(a, b):
+        assert (a == b) is False
+        assert (b == a) is False
+        assert (a != b) is True
+        assert (b != a) is True
+        assert (a < b) is True
+        assert (a <= b) is True
+        assert (a > b) is False
+        assert (a >= b) is False
+        assert (b < a) is False
+        assert (b <= a) is False
+        assert (b > a) is True
+        assert (b >= a) is True
+        assert hash(a) != hash(b)    # (or at least, it is unlikely)
+    def assert_gt(a, b):
+        assert_lt(b, a)
+    def assert_ne(a, b):
+        assert (a == b) is False
+        assert (b == a) is False
+        assert (a != b) is True
+        assert (b != a) is True
+        if strict_compare:
+            py.test.raises(TypeError, "a < b")
+            py.test.raises(TypeError, "a <= b")
+            py.test.raises(TypeError, "a > b")
+            py.test.raises(TypeError, "a >= b")
+            py.test.raises(TypeError, "b < a")
+            py.test.raises(TypeError, "b <= a")
+            py.test.raises(TypeError, "b > a")
+            py.test.raises(TypeError, "b >= a")
+        elif a < b:
+            assert_lt(a, b)
+        else:
+            assert_lt(b, a)
+    assert_eq(5, 5)
+    assert_lt(3, 5)
+    assert_ne('5', 5)
+    #
+    t1 = new_primitive_type("char")
+    t2 = new_primitive_type("int")
+    t3 = new_primitive_type("unsigned char")
+    t4 = new_primitive_type("unsigned int")
+    t5 = new_primitive_type("float")
+    t6 = new_primitive_type("double")
+    assert_eq(cast(t1, 65), b'A')
+    assert_lt(cast(t1, 64), b'\x99')
+    assert_gt(cast(t1, 200), b'A')
+    assert_ne(cast(t1, 65), 65)
+    assert_eq(cast(t2, -25), -25)
+    assert_lt(cast(t2, -25), -24)
+    assert_gt(cast(t2, -25), -26)
+    assert_eq(cast(t3, 65), 65)
+    assert_ne(cast(t3, 65), b'A')
+    assert_ne(cast(t3, 65), cast(t1, 65))
+    assert_gt(cast(t4, -1), -1)
+    assert_gt(cast(t4, -1), cast(t2, -1))
+    assert_gt(cast(t4, -1), 99999)
+    assert_eq(cast(t4, -1), 256 ** size_of_int() - 1)
+    assert_eq(cast(t5, 3.0), 3)
+    assert_eq(cast(t5, 3.5), 3.5)
+    assert_lt(cast(t5, 3.3), 3.3)   # imperfect rounding
+    assert_eq(cast(t6, 3.3), 3.3)
+    assert_eq(cast(t5, 3.5), cast(t6, 3.5))
+    assert_lt(cast(t5, 3.1), cast(t6, 3.1))   # imperfect rounding
+    assert_eq(cast(t5, 7.0), cast(t3, 7))
+    assert_lt(cast(t5, 3.1), 3.101)
+    assert_gt(cast(t5, 3.1), 3)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -55,7 +55,8 @@
         min = int(min)
         max = int(max)
         p = ffi.cast(c_decl, min)
-        assert p != min       # no __eq__(int)
+        assert p == min
+        assert hash(p) == hash(min)
         assert bool(p) is bool(min)
         assert int(p) == min
         p = ffi.cast(c_decl, max)
@@ -66,9 +67,9 @@
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
         q = ffi.cast(c_decl, long(min - 1))
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
-        assert q != p
+        assert q == p
         assert int(q) == int(p)
-        assert hash(q) != hash(p)   # unlikely
+        assert hash(q) == hash(p)
         c_decl_ptr = '%s *' % c_decl
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, min - 1)
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, max + 1)
@@ -883,9 +884,9 @@
         assert ffi.string(ffi.cast("enum bar", -2)) == "B1"
         assert ffi.string(ffi.cast("enum bar", -1)) == "CC1"
         assert ffi.string(ffi.cast("enum bar", 1)) == "E1"
-        assert ffi.cast("enum bar", -2) != ffi.cast("enum bar", -2)
-        assert ffi.cast("enum foo", 0) != ffi.cast("enum bar", 0)
-        assert ffi.cast("enum bar", 0) != ffi.cast("int", 0)
+        assert ffi.cast("enum bar", -2) == ffi.cast("enum bar", -2)
+        assert ffi.cast("enum foo", 0) == ffi.cast("enum bar", 0)
+        assert ffi.cast("enum bar", 0) == ffi.cast("int", 0)
         assert repr(ffi.cast("enum bar", -1)) == "<cdata 'enum bar' -1: CC1>"
         assert repr(ffi.cast("enum foo", -1)) == (  # enums are unsigned, if
             "<cdata 'enum foo' 4294967295>")        # they contain no neg value
@@ -1114,15 +1115,15 @@
         assert (q == None) is False
         assert (q != None) is True
 
-    def test_no_integer_comparison(self):
+    def test_integer_comparison(self):
         ffi = FFI(backend=self.Backend())
         x = ffi.cast("int", 123)
         y = ffi.cast("int", 456)
-        py.test.raises(TypeError, "x < y")
+        assert x < y
         #
         z = ffi.cast("double", 78.9)
-        py.test.raises(TypeError, "x < z")
-        py.test.raises(TypeError, "z < y")
+        assert x > z
+        assert y > z
 
     def test_ffi_buffer_ptr(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -138,7 +138,7 @@
         min = int(min)
         max = int(max)
         p = ffi.cast(c_decl, min)
-        assert p != min       # no __eq__(int)
+        assert p == min
         assert bool(p) is bool(min)
         assert int(p) == min
         p = ffi.cast(c_decl, max)
@@ -149,9 +149,9 @@
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
         q = ffi.cast(c_decl, long(min - 1))
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
-        assert q != p
+        assert q == p
         assert int(q) == int(p)
-        assert hash(q) != hash(p)   # unlikely
+        assert hash(q) == hash(p)
         c_decl_ptr = '%s *' % c_decl
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, min - 1)
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, max + 1)
@@ -897,9 +897,9 @@
         assert ffi.string(ffi.cast("enum bar", -2)) == "B1"
         assert ffi.string(ffi.cast("enum bar", -1)) == "CC1"
         assert ffi.string(ffi.cast("enum bar", 1)) == "E1"
-        assert ffi.cast("enum bar", -2) != ffi.cast("enum bar", -2)
-        assert ffi.cast("enum foq", 0) != ffi.cast("enum bar", 0)
-        assert ffi.cast("enum bar", 0) != ffi.cast("int", 0)
+        assert ffi.cast("enum bar", -2) == ffi.cast("enum bar", -2)
+        assert ffi.cast("enum foq", 0) == ffi.cast("enum bar", 0)
+        assert ffi.cast("enum bar", 0) == ffi.cast("int", 0)
         assert repr(ffi.cast("enum bar", -1)) == "<cdata 'enum bar' -1: CC1>"
         assert repr(ffi.cast("enum foq", -1)) == (  # enums are unsigned, if
             "<cdata 'enum foq' 4294967295>") or (   # they contain no neg value
@@ -1106,14 +1106,14 @@
         assert (q == None) is False
         assert (q != None) is True
 
-    def test_no_integer_comparison(self):
+    def test_integer_comparison(self):
         x = ffi.cast("int", 123)
         y = ffi.cast("int", 456)
-        py.test.raises(TypeError, "x < y")
+        assert x < y
         #
         z = ffi.cast("double", 78.9)
-        py.test.raises(TypeError, "x < z")
-        py.test.raises(TypeError, "z < y")
+        assert x > z
+        assert y > z
 
     def test_ffi_buffer_ptr(self):
         a = ffi.new("short *", 100)

From pypy.commits at gmail.com  Sun Feb 19 10:00:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:00:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test_surrogate_in_filename
Message-ID: <58a9b324.15502e0a.10271.1959@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90199:fea26f231a97
Date: 2017-02-19 16:00 +0100
http://bitbucket.org/pypy/pypy/changeset/fea26f231a97/

Log:	fix test_surrogate_in_filename

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -226,7 +226,7 @@
     return False
 
 def normalize_module(space, w_filename):
-    filename = space.identifier_w(w_filename)
+    filename = space.text_w(w_filename)
     if len(filename) == 0:
         return space.newtext("<unknown>")
     if filename.endswith(".py"):

From pypy.commits at gmail.com  Sun Feb 19 10:07:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:07:42 -0800 (PST)
Subject: [pypy-commit] pypy default: fix (for py3)
Message-ID: <58a9b4be.04162e0a.21a12.1e67@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90200:56126b34d84c
Date: 2017-02-19 16:04 +0100
http://bitbucket.org/pypy/pypy/changeset/56126b34d84c/

Log:	fix (for py3)

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -39,12 +39,12 @@
         arg = space.newtuple([one, one, one])
         # call w_date.__new__
         w_obj = space.call_function(w_date, one, one, one)
-        w_year = space.getattr(w_obj, space.newbytes('year'))
+        w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
         w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype,
                                  arg, space.newdict({}))
-        w_year = space.getattr(w_obj, space.newbytes('year'))
+        w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
 class AppTestUserSlots(AppTestCpythonExtensionBase):

From pypy.commits at gmail.com  Sun Feb 19 10:07:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:07:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a9b4c0.d9002e0a.25cf0.1fdc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90201:bd18926a5492
Date: 2017-02-19 16:05 +0100
http://bitbucket.org/pypy/pypy/changeset/bd18926a5492/

Log:	hg merge default

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -604,7 +604,8 @@
         """
         # hack for performance: if restype is a "simple" primitive type, don't
         # allocate the buffer because it's going to be thrown away immediately
-        if self._is_primitive(restype) and not restype._is_pointer_like():
+        if (self._is_primitive(restype) and restype._type_ != '?'
+            and not restype._is_pointer_like()):
             return result
         #
         shape = restype._ffishape_
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -112,11 +112,20 @@
     def _make_cmp(name):
         cmpfunc = getattr(operator, name)
         def cmp(self, other):
-            if isinstance(other, CTypesData):
+            v_is_ptr = not isinstance(self, CTypesGenericPrimitive)
+            w_is_ptr = (isinstance(other, CTypesData) and
+                           not isinstance(other, CTypesGenericPrimitive))
+            if v_is_ptr and w_is_ptr:
                 return cmpfunc(self._convert_to_address(None),
                                other._convert_to_address(None))
+            elif v_is_ptr or w_is_ptr:
+                return NotImplemented
             else:
-                return NotImplemented
+                if isinstance(self, CTypesGenericPrimitive):
+                    self = self._value
+                if isinstance(other, CTypesGenericPrimitive):
+                    other = other._value
+                return cmpfunc(self, other)
         cmp.func_name = name
         return cmp
 
@@ -128,7 +137,7 @@
     __ge__ = _make_cmp('__ge__')
 
     def __hash__(self):
-        return hash(type(self)) ^ hash(self._convert_to_address(None))
+        return hash(self._convert_to_address(None))
 
     def _to_string(self, maxlen):
         raise TypeError("string(): %r" % (self,))
@@ -137,14 +146,8 @@
 class CTypesGenericPrimitive(CTypesData):
     __slots__ = []
 
-    def __eq__(self, other):
-        return self is other
-
-    def __ne__(self, other):
-        return self is not other
-
     def __hash__(self):
-        return object.__hash__(self)
+        return hash(self._value)
 
     def _get_own_repr(self):
         return repr(self._from_ctypes(self._value))
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -99,27 +99,39 @@
         raise oefmt(space.w_TypeError,
                     "cdata of type '%s' has no len()", self.ctype.name)
 
+    def _compare_mode(self, w_other):
+        from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
+        zero = rffi.cast(lltype.Unsigned, 0)
+        self_is_ptr = not isinstance(self.ctype, W_CTypePrimitive)
+        other_is_ptr = (isinstance(w_other, W_CData) and
+                           not isinstance(w_other.ctype, W_CTypePrimitive))
+        if other_is_ptr and self_is_ptr:
+            with self as ptr1, w_other as ptr2:
+                ptr1 = rffi.cast(lltype.Unsigned, ptr1)
+                ptr2 = rffi.cast(lltype.Unsigned, ptr2)
+            return (0, ptr1, ptr2, None, None)
+        elif other_is_ptr or self_is_ptr:
+            return (-1, zero, zero, None, None)
+        else:
+            w_ob1 = self.convert_to_object()
+            if isinstance(w_other, W_CData):
+                w_ob2 = w_other.convert_to_object()
+            else:
+                w_ob2 = w_other
+            return (1, zero, zero, w_ob1, w_ob2)
+
     def _make_comparison(name):
         op = getattr(operator, name)
-        requires_ordering = name not in ('eq', 'ne')
         #
         def _cmp(self, w_other):
-            from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
             space = self.space
-            if not isinstance(w_other, W_CData):
+            mode, adr1, adr2, w_ob1, w_ob2 = self._compare_mode(w_other)
+            if mode == 0:
+                return space.newbool(op(adr1, adr2))
+            elif mode == 1:
+                return getattr(space, name)(w_ob1, w_ob2)
+            else:
                 return space.w_NotImplemented
-
-            with self as ptr1, w_other as ptr2:
-                if requires_ordering:
-                    if (isinstance(self.ctype, W_CTypePrimitive) or
-                        isinstance(w_other.ctype, W_CTypePrimitive)):
-                        raise oefmt(space.w_TypeError,
-                                    "cannot do comparison on a primitive "
-                                    "cdata")
-                    ptr1 = rffi.cast(lltype.Unsigned, ptr1)
-                    ptr2 = rffi.cast(lltype.Unsigned, ptr2)
-                result = op(ptr1, ptr2)
-            return space.newbool(result)
         #
         return func_with_new_name(_cmp, name)
 
@@ -131,6 +143,11 @@
     ge = _make_comparison('ge')
 
     def hash(self):
+        from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive
+        if isinstance(self.ctype, W_CTypePrimitive):
+            w_ob = self.convert_to_object()
+            if not isinstance(w_ob, W_CData):
+                return self.space.hash(w_ob)
         ptr = self.unsafe_escaping_ptr()
         h = rffi.cast(lltype.Signed, ptr)
         # To hash pointers in dictionaries.  Assumes that h shows some
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -16,6 +16,7 @@
                                        .replace(r'\\U', r'\U'))
     u = U()
     str2bytes = str
+    strict_compare = False
 else:
     type_or_class = "class"
     long = int
@@ -27,6 +28,7 @@
     bitem2bchr = bytechr
     u = ""
     str2bytes = lambda s: bytes(s, "ascii")
+    strict_compare = True
 
 def size_of_int():
     BInt = new_primitive_type("int")
@@ -95,11 +97,11 @@
     x = cast(p, -66 + (1<<199)*256)
     assert repr(x) == "<cdata 'signed char' -66>"
     assert int(x) == -66
-    assert (x == cast(p, -66)) is False
-    assert (x != cast(p, -66)) is True
+    assert (x == cast(p, -66)) is True
+    assert (x != cast(p, -66)) is False
     q = new_primitive_type("short")
-    assert (x == cast(q, -66)) is False
-    assert (x != cast(q, -66)) is True
+    assert (x == cast(q, -66)) is True
+    assert (x != cast(q, -66)) is False
 
 def test_sizeof_type():
     py.test.raises(TypeError, sizeof, 42.5)
@@ -164,7 +166,7 @@
             assert float(cast(p, 1.1)) != 1.1     # rounding error
             assert float(cast(p, 1E200)) == INF   # limited range
 
-        assert cast(p, -1.1) != cast(p, -1.1)
+        assert cast(p, -1.1) == cast(p, -1.1)
         assert repr(float(cast(p, -0.0))) == '-0.0'
         assert float(cast(p, b'\x09')) == 9.0
         assert float(cast(p, u+'\x09')) == 9.0
@@ -208,7 +210,7 @@
     p = new_primitive_type("char")
     assert bool(cast(p, 'A')) is True
     assert bool(cast(p, '\x00')) is False    # since 1.7
-    assert cast(p, '\x00') != cast(p, -17*256)
+    assert cast(p, '\x00') == cast(p, -17*256)
     assert int(cast(p, 'A')) == 65
     assert long(cast(p, 'A')) == 65
     assert type(int(cast(p, 'A'))) is int
@@ -371,23 +373,6 @@
     # that it is already loaded too, so it should work
     assert x.load_function(BVoidP, 'sqrt')
 
-def test_hash_differences():
-    BChar = new_primitive_type("char")
-    BInt = new_primitive_type("int")
-    BFloat = new_primitive_type("float")
-    for i in range(1, 20):
-        x1 = cast(BChar, chr(i))
-        x2 = cast(BInt, i)
-        if hash(x1) != hash(x2):
-            break
-    else:
-        raise AssertionError("hashes are equal")
-    for i in range(1, 20):
-        if hash(cast(BFloat, i)) != hash(float(i)):
-            break
-    else:
-        raise AssertionError("hashes are equal")
-
 def test_no_len_on_nonarray():
     p = new_primitive_type("int")
     py.test.raises(TypeError, len, cast(p, 42))
@@ -2250,12 +2235,17 @@
     BVoidP = new_pointer_type(new_void_type())
     p = newp(BIntP, 123)
     q = cast(BInt, 124)
-    py.test.raises(TypeError, "p < q")
-    py.test.raises(TypeError, "p <= q")
     assert (p == q) is False
     assert (p != q) is True
-    py.test.raises(TypeError, "p > q")
-    py.test.raises(TypeError, "p >= q")
+    assert (q == p) is False
+    assert (q != p) is True
+    if strict_compare:
+        py.test.raises(TypeError, "p < q")
+        py.test.raises(TypeError, "p <= q")
+        py.test.raises(TypeError, "q < p")
+        py.test.raises(TypeError, "q <= p")
+        py.test.raises(TypeError, "p > q")
+        py.test.raises(TypeError, "p >= q")
     r = cast(BVoidP, p)
     assert (p <  r) is False
     assert (p <= r) is True
@@ -3829,3 +3819,86 @@
         assert len(w) == 2
     # check that the warnings are associated with lines in this file
     assert w[1].lineno == w[0].lineno + 4
+
+def test_primitive_comparison():
+    def assert_eq(a, b):
+        assert (a == b) is True
+        assert (b == a) is True
+        assert (a != b) is False
+        assert (b != a) is False
+        assert (a < b) is False
+        assert (a <= b) is True
+        assert (a > b) is False
+        assert (a >= b) is True
+        assert (b < a) is False
+        assert (b <= a) is True
+        assert (b > a) is False
+        assert (b >= a) is True
+        assert hash(a) == hash(b)
+    def assert_lt(a, b):
+        assert (a == b) is False
+        assert (b == a) is False
+        assert (a != b) is True
+        assert (b != a) is True
+        assert (a < b) is True
+        assert (a <= b) is True
+        assert (a > b) is False
+        assert (a >= b) is False
+        assert (b < a) is False
+        assert (b <= a) is False
+        assert (b > a) is True
+        assert (b >= a) is True
+        assert hash(a) != hash(b)    # (or at least, it is unlikely)
+    def assert_gt(a, b):
+        assert_lt(b, a)
+    def assert_ne(a, b):
+        assert (a == b) is False
+        assert (b == a) is False
+        assert (a != b) is True
+        assert (b != a) is True
+        if strict_compare:
+            py.test.raises(TypeError, "a < b")
+            py.test.raises(TypeError, "a <= b")
+            py.test.raises(TypeError, "a > b")
+            py.test.raises(TypeError, "a >= b")
+            py.test.raises(TypeError, "b < a")
+            py.test.raises(TypeError, "b <= a")
+            py.test.raises(TypeError, "b > a")
+            py.test.raises(TypeError, "b >= a")
+        elif a < b:
+            assert_lt(a, b)
+        else:
+            assert_lt(b, a)
+    assert_eq(5, 5)
+    assert_lt(3, 5)
+    assert_ne('5', 5)
+    #
+    t1 = new_primitive_type("char")
+    t2 = new_primitive_type("int")
+    t3 = new_primitive_type("unsigned char")
+    t4 = new_primitive_type("unsigned int")
+    t5 = new_primitive_type("float")
+    t6 = new_primitive_type("double")
+    assert_eq(cast(t1, 65), b'A')
+    assert_lt(cast(t1, 64), b'\x99')
+    assert_gt(cast(t1, 200), b'A')
+    assert_ne(cast(t1, 65), 65)
+    assert_eq(cast(t2, -25), -25)
+    assert_lt(cast(t2, -25), -24)
+    assert_gt(cast(t2, -25), -26)
+    assert_eq(cast(t3, 65), 65)
+    assert_ne(cast(t3, 65), b'A')
+    assert_ne(cast(t3, 65), cast(t1, 65))
+    assert_gt(cast(t4, -1), -1)
+    assert_gt(cast(t4, -1), cast(t2, -1))
+    assert_gt(cast(t4, -1), 99999)
+    assert_eq(cast(t4, -1), 256 ** size_of_int() - 1)
+    assert_eq(cast(t5, 3.0), 3)
+    assert_eq(cast(t5, 3.5), 3.5)
+    assert_lt(cast(t5, 3.3), 3.3)   # imperfect rounding
+    assert_eq(cast(t6, 3.3), 3.3)
+    assert_eq(cast(t5, 3.5), cast(t6, 3.5))
+    assert_lt(cast(t5, 3.1), cast(t6, 3.1))   # imperfect rounding
+    assert_eq(cast(t5, 7.0), cast(t3, 7))
+    assert_lt(cast(t5, 3.1), 3.101)
+    assert_gt(cast(t5, 3.1), 3)
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -39,12 +39,12 @@
         arg = space.newtuple([one, one, one])
         # call w_date.__new__
         w_obj = space.call_function(w_date, one, one, one)
-        w_year = space.getattr(w_obj, space.newbytes('year'))
+        w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
         w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype,
                                  arg, space.newdict({}))
-        w_year = space.getattr(w_obj, space.newbytes('year'))
+        w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
 class AppTestUserSlots(AppTestCpythonExtensionBase):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -55,7 +55,8 @@
         min = int(min)
         max = int(max)
         p = ffi.cast(c_decl, min)
-        assert p != min       # no __eq__(int)
+        assert p == min
+        assert hash(p) == hash(min)
         assert bool(p) is bool(min)
         assert int(p) == min
         p = ffi.cast(c_decl, max)
@@ -66,9 +67,9 @@
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
         q = ffi.cast(c_decl, long(min - 1))
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
-        assert q != p
+        assert q == p
         assert int(q) == int(p)
-        assert hash(q) != hash(p)   # unlikely
+        assert hash(q) == hash(p)
         c_decl_ptr = '%s *' % c_decl
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, min - 1)
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, max + 1)
@@ -883,9 +884,9 @@
         assert ffi.string(ffi.cast("enum bar", -2)) == "B1"
         assert ffi.string(ffi.cast("enum bar", -1)) == "CC1"
         assert ffi.string(ffi.cast("enum bar", 1)) == "E1"
-        assert ffi.cast("enum bar", -2) != ffi.cast("enum bar", -2)
-        assert ffi.cast("enum foo", 0) != ffi.cast("enum bar", 0)
-        assert ffi.cast("enum bar", 0) != ffi.cast("int", 0)
+        assert ffi.cast("enum bar", -2) == ffi.cast("enum bar", -2)
+        assert ffi.cast("enum foo", 0) == ffi.cast("enum bar", 0)
+        assert ffi.cast("enum bar", 0) == ffi.cast("int", 0)
         assert repr(ffi.cast("enum bar", -1)) == "<cdata 'enum bar' -1: CC1>"
         assert repr(ffi.cast("enum foo", -1)) == (  # enums are unsigned, if
             "<cdata 'enum foo' 4294967295>")        # they contain no neg value
@@ -1114,15 +1115,15 @@
         assert (q == None) is False
         assert (q != None) is True
 
-    def test_no_integer_comparison(self):
+    def test_integer_comparison(self):
         ffi = FFI(backend=self.Backend())
         x = ffi.cast("int", 123)
         y = ffi.cast("int", 456)
-        py.test.raises(TypeError, "x < y")
+        assert x < y
         #
         z = ffi.cast("double", 78.9)
-        py.test.raises(TypeError, "x < z")
-        py.test.raises(TypeError, "z < y")
+        assert x > z
+        assert y > z
 
     def test_ffi_buffer_ptr(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -138,7 +138,7 @@
         min = int(min)
         max = int(max)
         p = ffi.cast(c_decl, min)
-        assert p != min       # no __eq__(int)
+        assert p == min
         assert bool(p) is bool(min)
         assert int(p) == min
         p = ffi.cast(c_decl, max)
@@ -149,9 +149,9 @@
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
         q = ffi.cast(c_decl, long(min - 1))
         assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
-        assert q != p
+        assert q == p
         assert int(q) == int(p)
-        assert hash(q) != hash(p)   # unlikely
+        assert hash(q) == hash(p)
         c_decl_ptr = '%s *' % c_decl
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, min - 1)
         py.test.raises(OverflowError, ffi.new, c_decl_ptr, max + 1)
@@ -897,9 +897,9 @@
         assert ffi.string(ffi.cast("enum bar", -2)) == "B1"
         assert ffi.string(ffi.cast("enum bar", -1)) == "CC1"
         assert ffi.string(ffi.cast("enum bar", 1)) == "E1"
-        assert ffi.cast("enum bar", -2) != ffi.cast("enum bar", -2)
-        assert ffi.cast("enum foq", 0) != ffi.cast("enum bar", 0)
-        assert ffi.cast("enum bar", 0) != ffi.cast("int", 0)
+        assert ffi.cast("enum bar", -2) == ffi.cast("enum bar", -2)
+        assert ffi.cast("enum foq", 0) == ffi.cast("enum bar", 0)
+        assert ffi.cast("enum bar", 0) == ffi.cast("int", 0)
         assert repr(ffi.cast("enum bar", -1)) == "<cdata 'enum bar' -1: CC1>"
         assert repr(ffi.cast("enum foq", -1)) == (  # enums are unsigned, if
             "<cdata 'enum foq' 4294967295>") or (   # they contain no neg value
@@ -1106,14 +1106,14 @@
         assert (q == None) is False
         assert (q != None) is True
 
-    def test_no_integer_comparison(self):
+    def test_integer_comparison(self):
         x = ffi.cast("int", 123)
         y = ffi.cast("int", 456)
-        py.test.raises(TypeError, "x < y")
+        assert x < y
         #
         z = ffi.cast("double", 78.9)
-        py.test.raises(TypeError, "x < z")
-        py.test.raises(TypeError, "z < y")
+        assert x > z
+        assert y > z
 
     def test_ffi_buffer_ptr(self):
         a = ffi.new("short *", 100)
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -99,6 +99,15 @@
         result = f(0, 0, 0, 0, 0, 0)
         assert result == b'\x00'
 
+    def test_boolresult(self):
+        f = dll._testfunc_i_bhilfd
+        f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+        f.restype = c_bool
+        false_result = f(0, 0, 0, 0, 0, 0)
+        assert false_result is False
+        true_result = f(1, 0, 0, 0, 0, 0)
+        assert true_result is True
+
     def test_voidresult(self):
         f = dll._testfunc_v
         f.restype = None

From pypy.commits at gmail.com  Sun Feb 19 10:14:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:14:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <58a9b66e.13a6190a.1b698.1a41@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90202:0e13d85aff3f
Date: 2017-02-19 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/0e13d85aff3f/

Log:	fix test

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -73,7 +73,10 @@
         # passed in, whose value is ignored. We don't implement that.
         #raises(IOError, _imp.create_dynamic, FakeSpec(), "unused")
 
-        raises(ImportError, _imp.create_dynamic, FakeSpec(b'foo'))
+        # Note: On CPython, the following gives nonsense.  I suspect
+        # it's because the b'foo' is read with PyUnicode_Xxx()
+        # functions that don't check the type of the argument.
+        raises(TypeError, _imp.create_dynamic, FakeSpec(b'foo'))
 
     def test_suffixes(self):
         import imp

From pypy.commits at gmail.com  Sun Feb 19 10:42:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:42:23 -0800 (PST)
Subject: [pypy-commit] pypy default: test and fix
Message-ID: <58a9bcdf.84472e0a.5e5fd.1e77@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90203:4334782fe642
Date: 2017-02-19 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/4334782fe642/

Log:	test and fix

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -378,7 +378,7 @@
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     else:
-        return space.newint(ok)
+        return space.newbool(ok)
 
 
 def times(space):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -368,11 +368,11 @@
         pdir = self.pdir + '/file1'
         posix = self.posix
 
-        assert posix.access(pdir, posix.R_OK)
-        assert posix.access(pdir, posix.W_OK)
+        assert posix.access(pdir, posix.R_OK) is True
+        assert posix.access(pdir, posix.W_OK) is True
         import sys
         if sys.platform != "win32":
-            assert not posix.access(pdir, posix.X_OK)
+            assert posix.access(pdir, posix.X_OK) is False
 
     def test_times(self):
         """

From pypy.commits at gmail.com  Sun Feb 19 10:42:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:42:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a9bce1.c916190a.72d58.1b84@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90204:13731a1c2af6
Date: 2017-02-19 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/13731a1c2af6/

Log:	hg merge default

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -621,7 +621,7 @@
     except OSError as e:
         raise wrap_oserror2(space, e, w_path, eintr_retry=False)
     else:
-        return space.newint(ok)
+        return space.newbool(ok)
 
 
 def times(space):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -362,11 +362,11 @@
         pdir = self.pdir + '/file1'
         posix = self.posix
 
-        assert posix.access(pdir, posix.R_OK)
-        assert posix.access(pdir, posix.W_OK)
+        assert posix.access(pdir, posix.R_OK) is True
+        assert posix.access(pdir, posix.W_OK) is True
         import sys
         if sys.platform != "win32":
-            assert not posix.access(pdir, posix.X_OK)
+            assert posix.access(pdir, posix.X_OK) is False
 
     def test_times(self):
         """

From pypy.commits at gmail.com  Sun Feb 19 10:54:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:54:49 -0800 (PST)
Subject: [pypy-commit] pypy default: Give up the fact that
 str.encode('utf-8') doesn't accept surrogates
Message-ID: <58a9bfc9.1c152e0a.642ad.13da@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90205:08b2dac171f5
Date: 2017-02-19 16:54 +0100
http://bitbucket.org/pypy/pypy/changeset/08b2dac171f5/

Log:	Give up the fact that str.encode('utf-8') doesn't accept surrogates
	in RPython. This is just asking for troubles because the same code
	accepts surrogates when run as plain Python (2.7).

diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -25,11 +25,7 @@
         assert value is not None
         errorhandler = runicode.default_unicode_error_decode
         u, pos = runicode.str_decode_utf_8_elidable(
-            value, len(value), 'strict', True, errorhandler, False)
-        # XXX should it really be 'allow_surrogates=False'?  In RPython,
-        # unicode.decode('utf-8') happily accepts surrogates.  This
-        # makes it hard to test untranslated (it's the cause of a
-        # failure in lib-python's test_warnings on PyPy3, for example)
+            value, len(value), 'strict', True, errorhandler, True)
         # XXX maybe the whole ''.decode('utf-8') should be not RPython.
         return self.ll.llunicode(u)
 
@@ -397,7 +393,7 @@
         errorhandler = runicode.default_unicode_error_encode
         bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict',
-            errorhandler=errorhandler, allow_surrogates=False)
+            errorhandler=errorhandler, allow_surrogates=True)
         return self.ll.llstr(bytes)
 
     def rtype_method_encode(self, hop):
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -110,7 +110,13 @@
             x = u'\ud800' + unichr(n)
             return x.encode('utf-8')
 
-        self.interpret_raises(UnicodeEncodeError, g, [38])
+        # used to raise in RPython, but not when run as plain Python,
+        # which just makes code very hard to test.  Nowadays, .encode()
+        # and .decode() accept surrogates like in Python 2.7.  Use
+        # functions from the rlib.runicode module if you need stricter
+        # behavior.
+        #self.interpret_raises(UnicodeEncodeError, g, [38])
+        assert self.ll_to_string(self.interpret(g, [38])) == g(38)
 
     def test_utf_8_encoding_annotation(self):
         from rpython.rlib.runicode import unicode_encode_utf_8

From pypy.commits at gmail.com  Sun Feb 19 10:56:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:56:35 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: spacing
Message-ID: <58a9c033.0163190a.47b76.77df@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90206:135243084607
Date: 2017-02-19 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/135243084607/

Log:	spacing

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1412,6 +1412,8 @@
         assert type.__ne__(dict, 42) is NotImplemented
         assert type.__eq__(int, int) == True
         assert type.__eq__(int, dict) is NotImplemented
+
+
 class AppTestComparesByIdentity:
 
     def setup_class(cls):

From pypy.commits at gmail.com  Sun Feb 19 10:56:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 07:56:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a9c036.501d190a.fc809.1c91@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90207:6ea6e99c6940
Date: 2017-02-19 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/6ea6e99c6940/

Log:	hg merge default

diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -25,11 +25,7 @@
         assert value is not None
         errorhandler = runicode.default_unicode_error_decode
         u, pos = runicode.str_decode_utf_8_elidable(
-            value, len(value), 'strict', True, errorhandler, False)
-        # XXX should it really be 'allow_surrogates=False'?  In RPython,
-        # unicode.decode('utf-8') happily accepts surrogates.  This
-        # makes it hard to test untranslated (it's the cause of a
-        # failure in lib-python's test_warnings on PyPy3, for example)
+            value, len(value), 'strict', True, errorhandler, True)
         # XXX maybe the whole ''.decode('utf-8') should be not RPython.
         return self.ll.llunicode(u)
 
@@ -397,7 +393,7 @@
         errorhandler = runicode.default_unicode_error_encode
         bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict',
-            errorhandler=errorhandler, allow_surrogates=False)
+            errorhandler=errorhandler, allow_surrogates=True)
         return self.ll.llstr(bytes)
 
     def rtype_method_encode(self, hop):
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -110,7 +110,13 @@
             x = u'\ud800' + unichr(n)
             return x.encode('utf-8')
 
-        self.interpret_raises(UnicodeEncodeError, g, [38])
+        # used to raise in RPython, but not when run as plain Python,
+        # which just makes code very hard to test.  Nowadays, .encode()
+        # and .decode() accept surrogates like in Python 2.7.  Use
+        # functions from the rlib.runicode module if you need stricter
+        # behavior.
+        #self.interpret_raises(UnicodeEncodeError, g, [38])
+        assert self.ll_to_string(self.interpret(g, [38])) == g(38)
 
     def test_utf_8_encoding_annotation(self):
         from rpython.rlib.runicode import unicode_encode_utf_8

From pypy.commits at gmail.com  Sun Feb 19 11:07:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 08:07:10 -0800 (PST)
Subject: [pypy-commit] pypy default: translation fix (mostly for tests)
Message-ID: <58a9c2ae.11162e0a.87141.150e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90208:3b6d1f077a95
Date: 2017-02-19 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/3b6d1f077a95/

Log:	translation fix (mostly for tests)

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -107,6 +107,12 @@
         return u'', None, endingpos
     raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
+def ll_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos):
+    raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
+
+def ll_unicode_error_encode(errors, encoding, msg, u, startingpos, endingpos):
+    raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
+
 # ____________________________________________________________
 # utf-8
 
@@ -143,8 +149,8 @@
         size = NonConstant(12345)
         errors = NonConstant('strict')
         final = NonConstant(True)
-        errorhandler = default_unicode_error_decode
-        allow_surrogates = NonConstant(False)
+        errorhandler = ll_unicode_error_decode
+        allow_surrogates = NonConstant(True)
     return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
@@ -359,8 +365,8 @@
         s = NonConstant(u'?????')
         size = NonConstant(12345)
         errors = NonConstant('strict')
-        errorhandler = default_unicode_error_encode
-        allow_surrogates = NonConstant(False)
+        errorhandler = ll_unicode_error_encode
+        allow_surrogates = NonConstant(True)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
 
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -23,7 +23,7 @@
         from rpython.rlib import runicode
         value = hlstr(llvalue)
         assert value is not None
-        errorhandler = runicode.default_unicode_error_decode
+        errorhandler = runicode.ll_unicode_error_decode
         u, pos = runicode.str_decode_utf_8_elidable(
             value, len(value), 'strict', True, errorhandler, True)
         # XXX maybe the whole ''.decode('utf-8') should be not RPython.
@@ -390,7 +390,7 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
-        errorhandler = runicode.default_unicode_error_encode
+        errorhandler = runicode.ll_unicode_error_encode
         bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict',
             errorhandler=errorhandler, allow_surrogates=True)

From pypy.commits at gmail.com  Sun Feb 19 11:07:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 08:07:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a9c2b0.1427190a.42fb.1803@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90209:d9cf694006ab
Date: 2017-02-19 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/d9cf694006ab/

Log:	hg merge default

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -107,6 +107,12 @@
         return u'', None, endingpos
     raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
+def ll_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos):
+    raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
+
+def ll_unicode_error_encode(errors, encoding, msg, u, startingpos, endingpos):
+    raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
+
 # ____________________________________________________________
 # utf-8
 
@@ -143,8 +149,8 @@
         size = NonConstant(12345)
         errors = NonConstant('strict')
         final = NonConstant(True)
-        errorhandler = default_unicode_error_decode
-        allow_surrogates = NonConstant(False)
+        errorhandler = ll_unicode_error_decode
+        allow_surrogates = NonConstant(True)
     return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
@@ -359,8 +365,8 @@
         s = NonConstant(u'?????')
         size = NonConstant(12345)
         errors = NonConstant('strict')
-        errorhandler = default_unicode_error_encode
-        allow_surrogates = NonConstant(False)
+        errorhandler = ll_unicode_error_encode
+        allow_surrogates = NonConstant(True)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
 
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -23,7 +23,7 @@
         from rpython.rlib import runicode
         value = hlstr(llvalue)
         assert value is not None
-        errorhandler = runicode.default_unicode_error_decode
+        errorhandler = runicode.ll_unicode_error_decode
         u, pos = runicode.str_decode_utf_8_elidable(
             value, len(value), 'strict', True, errorhandler, True)
         # XXX maybe the whole ''.decode('utf-8') should be not RPython.
@@ -390,7 +390,7 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
-        errorhandler = runicode.default_unicode_error_encode
+        errorhandler = runicode.ll_unicode_error_encode
         bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict',
             errorhandler=errorhandler, allow_surrogates=True)

From pypy.commits at gmail.com  Sun Feb 19 11:46:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 08:46:04 -0800 (PST)
Subject: [pypy-commit] pypy default: reduce the footprint of this table
Message-ID: <58a9cbcc.0b202e0a.93a0a.22b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90210:952e978c7009
Date: 2017-02-19 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/952e978c7009/

Log:	reduce the footprint of this table

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -116,15 +116,7 @@
 # ____________________________________________________________
 # utf-8
 
-utf8_code_length = [
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, # 00-0F
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, # 70-7F
+_utf8_code_length = ''.join(map(chr, [
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 80-8F
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -133,7 +125,7 @@
     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, # D0-DF
     3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, # E0-EF
     4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0  # F0-F4 - F5-FF
-]
+]))
 
 # if you can't use the @elidable version, call str_decode_utf_8_impl()
 # directly
@@ -190,7 +182,7 @@
             pos += 1
             continue
 
-        n = utf8_code_length[ordch1]
+        n = ord(_utf8_code_length[ordch1 - 0x80])
         if pos + n > size:
             if not final:
                 break

From pypy.commits at gmail.com  Sun Feb 19 12:47:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 09:47:36 -0800 (PST)
Subject: [pypy-commit] pypy default: more attempts to fix translation errors
Message-ID: <58a9da38.21d8190a.50f32.243d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90211:ecec05ae43ec
Date: 2017-02-19 18:43 +0100
http://bitbucket.org/pypy/pypy/changeset/ecec05ae43ec/

Log:	more attempts to fix translation errors

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -699,7 +699,7 @@
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for unicode" % (enc,))
         return SomeString(no_nul=self.no_nul)
-    method_encode.can_only_throw = [UnicodeEncodeError]
+    method_encode.can_only_throw = []
 
 
 class __extend__(SomeString):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -110,9 +110,6 @@
 def ll_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos):
     raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
 
-def ll_unicode_error_encode(errors, encoding, msg, u, startingpos, endingpos):
-    raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
-
 # ____________________________________________________________
 # utf-8
 
@@ -357,7 +354,7 @@
         s = NonConstant(u'?????')
         size = NonConstant(12345)
         errors = NonConstant('strict')
-        errorhandler = ll_unicode_error_encode
+        # no errorhandler needed for rtyper/rstr.py
         allow_surrogates = NonConstant(True)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -390,10 +390,8 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
-        errorhandler = runicode.ll_unicode_error_encode
         bytes = runicode.unicode_encode_utf_8_elidable(
-            s, len(s), 'strict',
-            errorhandler=errorhandler, allow_surrogates=True)
+            s, len(s), 'strict', None, True)
         return self.ll.llstr(bytes)
 
     def rtype_method_encode(self, hop):

From pypy.commits at gmail.com  Sun Feb 19 12:47:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 09:47:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: typo
Message-ID: <58a9da3a.1e142e0a.32fa4.23b1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90212:aabda7c4125f
Date: 2017-02-19 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/aabda7c4125f/

Log:	typo

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -379,7 +379,7 @@
         assert bytes(b2) == self.struct.pack("ii", 17, 42) + (b'\x00' * 11)
 
         exc = raises(TypeError, self.struct.pack_into, "ii", b'test', 0, 17, 42)
-        assert str(exc.value) == "a read-write buffer is requried, not bytes"
+        assert str(exc.value) == "a read-write buffer is required, not bytes"
         exc = raises(self.struct.error, self.struct.pack_into, "ii", b[0:1], 0, 17, 42)
         assert str(exc.value) == "pack_into requires a buffer of at least 8 bytes"
 

From pypy.commits at gmail.com  Sun Feb 19 12:47:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 09:47:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58a9da3d.0b202e0a.93a0a.25c9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90213:691bc97b8dc9
Date: 2017-02-19 18:47 +0100
http://bitbucket.org/pypy/pypy/changeset/691bc97b8dc9/

Log:	hg merge default

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -699,7 +699,7 @@
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for unicode" % (enc,))
         return SomeString(no_nul=self.no_nul)
-    method_encode.can_only_throw = [UnicodeEncodeError]
+    method_encode.can_only_throw = []
 
 
 class __extend__(SomeString):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -110,21 +110,10 @@
 def ll_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos):
     raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
 
-def ll_unicode_error_encode(errors, encoding, msg, u, startingpos, endingpos):
-    raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
-
 # ____________________________________________________________
 # utf-8
 
-utf8_code_length = [
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, # 00-0F
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, # 70-7F
+_utf8_code_length = ''.join(map(chr, [
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 80-8F
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -133,7 +122,7 @@
     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, # D0-DF
     3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, # E0-EF
     4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0  # F0-F4 - F5-FF
-]
+]))
 
 # if you can't use the @elidable version, call str_decode_utf_8_impl()
 # directly
@@ -190,7 +179,7 @@
             pos += 1
             continue
 
-        n = utf8_code_length[ordch1]
+        n = ord(_utf8_code_length[ordch1 - 0x80])
         if pos + n > size:
             if not final:
                 break
@@ -365,7 +354,7 @@
         s = NonConstant(u'?????')
         size = NonConstant(12345)
         errors = NonConstant('strict')
-        errorhandler = ll_unicode_error_encode
+        # no errorhandler needed for rtyper/rstr.py
         allow_surrogates = NonConstant(True)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -390,10 +390,8 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
-        errorhandler = runicode.ll_unicode_error_encode
         bytes = runicode.unicode_encode_utf_8_elidable(
-            s, len(s), 'strict',
-            errorhandler=errorhandler, allow_surrogates=True)
+            s, len(s), 'strict', None, True)
         return self.ll.llstr(bytes)
 
     def rtype_method_encode(self, hop):

From pypy.commits at gmail.com  Sun Feb 19 14:04:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 19 Feb 2017 11:04:54 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58a9ec56.14542e0a.f83e2.1d74@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r864:e44365234a13
Date: 2017-02-19 20:04 +0100
http://bitbucket.org/pypy/pypy.org/changeset/e44365234a13/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66639 of $105000 (63.5%)
+   $66648 of $105000 (63.5%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2461 left
+    we have $2469 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sun Feb 19 15:50:49 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 19 Feb 2017 12:50:49 -0800 (PST)
Subject: [pypy-commit] pypy default: remove two wraps and a str_w
Message-ID: <58aa0529.554c190a.c616c.2ca6@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r90214:408b2ec34a3f
Date: 2017-02-19 21:49 +0100
http://bitbucket.org/pypy/pypy/changeset/408b2ec34a3f/

Log:	remove two wraps and a str_w

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1105,7 +1105,7 @@
                     raise
                 # UnicodeError in literal: turn into SyntaxError
                 e.normalize_exception(space)
-                errmsg = space.str_w(space.str(e.get_w_value(space)))
+                errmsg = space.text_w(space.str(e.get_w_value(space)))
                 raise self.error('(unicode error) %s' % errmsg, atom_node)
             # This implements implicit string concatenation.
             if len(sub_strings_w) > 1:
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -26,7 +26,7 @@
 
 @slot_function([PyObject, Py_ssize_t], PyObject)
 def slot_sq_item(space, w_obj, index):
-    return space.getitem(w_obj, space.wrap(index))
+    return space.getitem(w_obj, space.newint(index))
 
 @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
 def slot_tp_new(space, w_type, w_args, w_kwds):
@@ -39,12 +39,12 @@
     #     we know (since we are in this function) that self is not a cpytype
     from pypy.module.cpyext.typeobject import W_PyCTypeObject
     w_type0 = w_type
-    mro_w = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    mro_w = space.listview(space.getattr(w_type0, space.newtext('__mro__')))
     for w_m in mro_w[1:]:
         if not w_type0.is_cpytype():
             break
         w_type0 = w_m
-    w_impl = space.getattr(w_type0, space.wrap('__new__'))
+    w_impl = space.getattr(w_type0, space.newtext('__new__'))
     args = Arguments(space, [w_type],
                      w_stararg=w_args, w_starstararg=w_kwds)
     return space.call_args(w_impl, args)

From pypy.commits at gmail.com  Sun Feb 19 20:52:36 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Sun, 19 Feb 2017 17:52:36 -0800 (PST)
Subject: [pypy-commit] cffi sirtom67/float_complex: "c" tests pass!
Message-ID: <58aa4be4.51d2190a.c217d.2c63@mx.google.com>

Author: Tom Krauss <thomas.p.krauss at gmail.com>
Branch: sirtom67/float_complex
Changeset: r2891:3ce9e47d35f0
Date: 2017-02-19 19:51 -0600
http://bitbucket.org/cffi/cffi/changeset/3ce9e47d35f0/

Log:	"c" tests pass!

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3536,6 +3536,36 @@
     return cd;
 }
 
+static void check_bytes_for_float_compatible(
+        PyObject *io,
+        double * value,
+        int * got_value_indicator,
+        int * cannot_cast_indicator)
+{
+    *got_value_indicator = 0;
+    *cannot_cast_indicator = 0;
+    if (PyBytes_Check(io)) {
+        if (PyBytes_GET_SIZE(io) != 1) {
+            Py_DECREF(io);
+            *cannot_cast_indicator = 1;
+        }
+        *value = (unsigned char)PyBytes_AS_STRING(io)[0];
+        *got_value_indicator = 1;
+    }
+#if HAVE_WCHAR_H
+    else if (PyUnicode_Check(io)) {
+        wchar_t ordinal;
+        if (_my_PyUnicode_AsSingleWideChar(io, &ordinal) < 0) {
+            Py_DECREF(io);
+            *cannot_cast_indicator = 1;
+        }
+        *value = (long)ordinal;
+        *got_value_indicator = 1;
+    }
+#endif
+
+}
+
 static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob)
 {
     CDataObject *cd;
@@ -3592,23 +3622,16 @@
             Py_INCREF(io);
         }
 
-        if (PyBytes_Check(io)) {
-            if (PyBytes_GET_SIZE(io) != 1) {
-                Py_DECREF(io);
-                goto cannot_cast;
-            }
-            value = (unsigned char)PyBytes_AS_STRING(io)[0];
-        }
-#if HAVE_WCHAR_H
-        else if (PyUnicode_Check(io)) {
-            wchar_t ordinal;
-            if (_my_PyUnicode_AsSingleWideChar(io, &ordinal) < 0) {
-                Py_DECREF(io);
-                goto cannot_cast;
-            }
-            value = (long)ordinal;
-        }
-#endif
+        int got_value_indicator;
+        int cannot_cast_indicator;
+        check_bytes_for_float_compatible(io, &value,
+                &got_value_indicator, &cannot_cast_indicator);
+        if (cannot_cast_indicator) {
+            goto cannot_cast;
+        }
+        if (got_value_indicator) {
+            // got it from string
+        }
         else if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
                  CData_Check(io) &&
                  (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
@@ -3658,11 +3681,23 @@
             Py_INCREF(io);
         }
 
-        value = PyComplex_AsCComplex(io);
+        int got_value_indicator;
+        int cannot_cast_indicator;
+        check_bytes_for_float_compatible(io, &(value.real),
+                &got_value_indicator, &cannot_cast_indicator);
+        if (cannot_cast_indicator) {
+            goto cannot_cast;
+        }
+        if (got_value_indicator) {
+            // got it from string
+            value.imag = 0.0;
+        } else {
+            value = PyComplex_AsCComplex(io);
+        }
         Py_DECREF(io);
-        if (value.real == -1.0 && value.imag == 0.0 && PyErr_Occurred())
+        if (value.real == -1.0 && value.imag == 0.0 && PyErr_Occurred()) {
             return NULL;
-
+        }
         cd = _new_casted_primitive(ct);
         if (cd != NULL) {
             write_raw_complex_data(cd->c_data, value, ct->ct_size);
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -206,8 +206,9 @@
 
         assert cast(p, -1.1j) != cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        assert repr(complex(cast(p, -0j))) == '-0j'
-        assert complex(cast(p, '\x09')) == 9.0
+        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
+        assert complex(cast(p, b'\x09')) == 9.0
+        assert complex(cast(p, u+'\x09')) == 9.0
         assert complex(cast(p, True)) == 1.0
         py.test.raises(TypeError, cast, p, None)
         #

From pypy.commits at gmail.com  Mon Feb 20 03:12:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 00:12:32 -0800 (PST)
Subject: [pypy-commit] pypy default: Pff,
 whack more at the hack for calling str_encode_utf_8 from
Message-ID: <58aaa4f0.51072e0a.d002.ce89@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90215:bde34ec1885f
Date: 2017-02-20 09:10 +0100
http://bitbucket.org/pypy/pypy/changeset/bde34ec1885f/

Log:	Pff, whack more at the hack for calling str_encode_utf_8 from
	''.encode('utf-8')

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -698,8 +698,18 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for unicode" % (enc,))
+        if enc == 'utf-8':
+            from rpython.rlib import runicode
+            bookkeeper = getbookkeeper()
+            s_func = bookkeeper.immutablevalue(
+                             runicode.unicode_encode_utf_8_elidable)
+            s_errors = bookkeeper.immutablevalue('strict')
+            s_errorhandler = bookkeeper.immutablevalue(None)
+            s_allow_surr = bookkeeper.immutablevalue(True)
+            args = [self, self.len(), s_errors, s_errorhandler, s_allow_surr]
+            bookkeeper.emulate_pbc_call(bookkeeper.position_key, s_func, args)
         return SomeString(no_nul=self.no_nul)
-    method_encode.can_only_throw = []
+    method_encode.can_only_throw = [UnicodeEncodeError]
 
 
 class __extend__(SomeString):
@@ -731,6 +741,19 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for strings" % (enc,))
+        if enc == 'utf-8':
+            from rpython.rlib import runicode
+            bookkeeper = getbookkeeper()
+            s_func = bookkeeper.immutablevalue(
+                            runicode.str_decode_utf_8_elidable)
+            s_errors = bookkeeper.immutablevalue('strict')
+            s_final = bookkeeper.immutablevalue(True)
+            s_errorhandler = bookkeeper.immutablevalue(
+                                    runicode.default_unicode_error_decode)
+            s_allow_surr = bookkeeper.immutablevalue(True)
+            args = [self, self.len(), s_errors, s_final, s_errorhandler,
+                    s_allow_surr]
+            bookkeeper.emulate_pbc_call(bookkeeper.position_key, s_func, args)
         return SomeUnicodeString(no_nul=self.no_nul)
     method_decode.can_only_throw = [UnicodeDecodeError]
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -6,7 +6,6 @@
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import jit
-from rpython.rlib.nonconst import NonConstant
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -107,9 +106,6 @@
         return u'', None, endingpos
     raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
-def ll_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos):
-    raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
-
 # ____________________________________________________________
 # utf-8
 
@@ -131,15 +127,6 @@
                      errorhandler=None, allow_surrogates=allow_surrogate_by_default):
     if errorhandler is None:
         errorhandler = default_unicode_error_decode
-    # NB. a bit messy because rtyper/rstr.py also calls the same
-    # function.  Make sure we annotate for the args it passes, too
-    if NonConstant(False):
-        s = NonConstant('?????')
-        size = NonConstant(12345)
-        errors = NonConstant('strict')
-        final = NonConstant(True)
-        errorhandler = ll_unicode_error_decode
-        allow_surrogates = NonConstant(True)
     return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
@@ -348,14 +335,6 @@
     #
     if errorhandler is None:
         errorhandler = default_unicode_error_encode
-    # NB. a bit messy because rtyper/rstr.py also calls the same
-    # function.  Make sure we annotate for the args it passes, too
-    if NonConstant(False):
-        s = NonConstant(u'?????')
-        size = NonConstant(12345)
-        errors = NonConstant('strict')
-        # no errorhandler needed for rtyper/rstr.py
-        allow_surrogates = NonConstant(True)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
 
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -13,17 +13,14 @@
 
 class AbstractStringRepr(Repr):
 
-    def __init__(self, *args):
-        Repr.__init__(self, *args)
-        self.rstr_decode_utf_8 = None
-
     @jit.elidable
     def ll_decode_utf8(self, llvalue):
         from rpython.rtyper.annlowlevel import hlstr
         from rpython.rlib import runicode
         value = hlstr(llvalue)
         assert value is not None
-        errorhandler = runicode.ll_unicode_error_decode
+        errorhandler = runicode.default_unicode_error_decode
+        # NB. keep the arguments in sync with annotator/unaryop.py
         u, pos = runicode.str_decode_utf_8_elidable(
             value, len(value), 'strict', True, errorhandler, True)
         # XXX maybe the whole ''.decode('utf-8') should be not RPython.
@@ -374,10 +371,6 @@
 
 class AbstractUnicodeRepr(AbstractStringRepr):
 
-    def __init__(self, *args):
-        AbstractStringRepr.__init__(self, *args)
-        self.runicode_encode_utf_8 = None
-
     def rtype_method_upper(self, hop):
         raise TyperError("Cannot do toupper on unicode string")
 
@@ -390,6 +383,7 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
+        # NB. keep the arguments in sync with annotator/unaryop.py
         bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict', None, True)
         return self.ll.llstr(bytes)

From pypy.commits at gmail.com  Mon Feb 20 03:12:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 00:12:48 -0800 (PST)
Subject: [pypy-commit] pypy default: document pulled PR
Message-ID: <58aaa500.16502e0a.696c.3486@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90216:ca7a2e7283e5
Date: 2017-02-20 09:12 +0100
http://bitbucket.org/pypy/pypy/changeset/ca7a2e7283e5/

Log:	document pulled PR

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -150,3 +150,7 @@
 and ``space.newtext()``: these two methods are identical on PyPy 2.7 but
 not on PyPy 3.x.  The latter is used to get an app-level unicode string
 by decoding the RPython string, assumed to be utf-8.
+
+.. branch: fix_bool_restype
+
+Fix for ``ctypes.c_bool``-returning ctypes functions

From pypy.commits at gmail.com  Mon Feb 20 03:17:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 00:17:39 -0800 (PST)
Subject: [pypy-commit] cffi default: 32-bit fix
Message-ID: <58aaa623.c80e1c0a.e808e.e155@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2892:26883ba44f00
Date: 2017-02-20 09:17 +0100
http://bitbucket.org/cffi/cffi/changeset/26883ba44f00/

Log:	32-bit fix

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3846,7 +3846,7 @@
         assert (b > a) is False
         assert (b >= a) is True
         assert hash(a) == hash(b)
-    def assert_lt(a, b):
+    def assert_lt(a, b, check_hash=True):
         assert (a == b) is False
         assert (b == a) is False
         assert (a != b) is True
@@ -3859,9 +3859,10 @@
         assert (b <= a) is False
         assert (b > a) is True
         assert (b >= a) is True
-        assert hash(a) != hash(b)    # (or at least, it is unlikely)
-    def assert_gt(a, b):
-        assert_lt(b, a)
+        if check_hash:
+            assert hash(a) != hash(b)    # (or at least, it is unlikely)
+    def assert_gt(a, b, check_hash=True):
+        assert_lt(b, a, check_hash)
     def assert_ne(a, b):
         assert (a == b) is False
         assert (b == a) is False
@@ -3900,8 +3901,8 @@
     assert_eq(cast(t3, 65), 65)
     assert_ne(cast(t3, 65), b'A')
     assert_ne(cast(t3, 65), cast(t1, 65))
-    assert_gt(cast(t4, -1), -1)
-    assert_gt(cast(t4, -1), cast(t2, -1))
+    assert_gt(cast(t4, -1), -1, check_hash=False)
+    assert_gt(cast(t4, -1), cast(t2, -1), check_hash=False)
     assert_gt(cast(t4, -1), 99999)
     assert_eq(cast(t4, -1), 256 ** size_of_int() - 1)
     assert_eq(cast(t5, 3.0), 3)

From pypy.commits at gmail.com  Mon Feb 20 03:24:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 00:24:16 -0800 (PST)
Subject: [pypy-commit] pypy default: import cffi/26883ba44f00
Message-ID: <58aaa7b0.53be190a.202e0.7786@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90217:fd2d7f445ec1
Date: 2017-02-20 09:17 +0100
http://bitbucket.org/pypy/pypy/changeset/fd2d7f445ec1/

Log:	import cffi/26883ba44f00

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3835,7 +3835,7 @@
         assert (b > a) is False
         assert (b >= a) is True
         assert hash(a) == hash(b)
-    def assert_lt(a, b):
+    def assert_lt(a, b, check_hash=True):
         assert (a == b) is False
         assert (b == a) is False
         assert (a != b) is True
@@ -3848,9 +3848,10 @@
         assert (b <= a) is False
         assert (b > a) is True
         assert (b >= a) is True
-        assert hash(a) != hash(b)    # (or at least, it is unlikely)
-    def assert_gt(a, b):
-        assert_lt(b, a)
+        if check_hash:
+            assert hash(a) != hash(b)    # (or at least, it is unlikely)
+    def assert_gt(a, b, check_hash=True):
+        assert_lt(b, a, check_hash)
     def assert_ne(a, b):
         assert (a == b) is False
         assert (b == a) is False
@@ -3889,8 +3890,8 @@
     assert_eq(cast(t3, 65), 65)
     assert_ne(cast(t3, 65), b'A')
     assert_ne(cast(t3, 65), cast(t1, 65))
-    assert_gt(cast(t4, -1), -1)
-    assert_gt(cast(t4, -1), cast(t2, -1))
+    assert_gt(cast(t4, -1), -1, check_hash=False)
+    assert_gt(cast(t4, -1), cast(t2, -1), check_hash=False)
     assert_gt(cast(t4, -1), 99999)
     assert_eq(cast(t4, -1), 256 ** size_of_int() - 1)
     assert_eq(cast(t5, 3.0), 3)

From pypy.commits at gmail.com  Mon Feb 20 04:06:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 01:06:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test and fix
Message-ID: <58aab1a5.59142e0a.af4fc.430f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90218:3910e862f7e9
Date: 2017-02-20 10:06 +0100
http://bitbucket.org/pypy/pypy/changeset/3910e862f7e9/

Log:	test and fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -93,8 +93,7 @@
             if space.is_w(w_result, space.w_None):
                 raise OperationError(space.w_StopIteration, space.w_None)
             else:
-                raise OperationError(space.w_StopIteration,
-                        space.call_function(space.w_StopIteration, w_result))
+                raise stopiteration_value(space, w_result)
         else:
             return w_result     # YIELDed
 
@@ -442,7 +441,14 @@
         return self
 
     def descr__next__(self, space):
-        raise OperationError(space.w_StopIteration, self.w_aiter)
+        raise stopiteration_value(space, self.w_aiter)
+
+def stopiteration_value(space, w_value):
+    # Mess.  The obvious line, "OperationError(w_StopIteration, w_value)",
+    # fails for some types of w_value.  E.g. if it's a subclass of
+    # tuple, it will be unpacked as individual arguments.
+    raise OperationError(space.w_StopIteration,
+                         space.call_function(space.w_StopIteration, w_value))
 
 
 @specialize.memo()
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -177,3 +177,29 @@
         assert str(w).startswith("coroutine ")
         assert str(w).endswith("foobaz' was never awaited")
         """
+
+    def test_async_for_with_tuple_subclass(self): """
+        class Done(Exception): pass
+
+        class AIter(tuple):
+            i = 0
+            def __aiter__(self):
+                return self
+            async def __anext__(self):
+                if self.i >= len(self):
+                    raise StopAsyncIteration
+                self.i += 1
+                return self[self.i - 1]
+
+        result = []
+        async def foo():
+            async for i in AIter([42]):
+                result.append(i)
+            raise Done
+
+        try:
+            foo().send(None)
+        except Done:
+            pass
+        assert result == [42]
+        """

From pypy.commits at gmail.com  Mon Feb 20 04:44:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 01:44:08 -0800 (PST)
Subject: [pypy-commit] pypy default: Add a __contains__ method on
 dict.viewkeys() and dict.viewitems().
Message-ID: <58aaba68.8521190a.1ec22.479c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90219:d9ee4fe1cd44
Date: 2017-02-20 10:43 +0100
http://bitbucket.org/pypy/pypy/changeset/d9ee4fe1cd44/

Log:	Add a __contains__ method on dict.viewkeys() and dict.viewitems().
	Without it, it would walk the list, which is the wrong complexity

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1585,9 +1585,25 @@
     def descr_iter(self, space):
         return W_DictMultiIterItemsObject(space, self.w_dict.iteritems())
 
+    def descr_contains(self, space, w_item):
+        if not space.isinstance_w(w_item, space.w_tuple):
+            return space.w_False
+        try:
+            w_key, w_value = space.fixedview_unroll(w_item, 2)
+            w_found = self.w_dict.getitem(w_key)
+        except OperationError as e:
+            if e.async(space):
+                raise
+            w_found = None
+        if w_found is None:
+            return space.w_False
+        return space.eq(w_value, w_found)
+
 class W_DictViewKeysObject(W_DictViewObject, SetLikeDictView):
     def descr_iter(self, space):
         return W_DictMultiIterKeysObject(space, self.w_dict.iterkeys())
+    def descr_contains(self, space, w_key):
+        return self.w_dict.descr_contains(space, w_key)
 
 class W_DictViewValuesObject(W_DictViewObject):
     def descr_iter(self, space):
@@ -1598,6 +1614,7 @@
     __repr__ = interp2app(W_DictViewItemsObject.descr_repr),
     __len__ = interp2app(W_DictViewItemsObject.descr_len),
     __iter__ = interp2app(W_DictViewItemsObject.descr_iter),
+    __contains__ = interp2app(W_DictViewItemsObject.descr_contains),
 
     __eq__ = interp2app(W_DictViewItemsObject.descr_eq),
     __ne__ = interp2app(W_DictViewItemsObject.descr_ne),
@@ -1621,6 +1638,7 @@
     __repr__ = interp2app(W_DictViewKeysObject.descr_repr),
     __len__ = interp2app(W_DictViewKeysObject.descr_len),
     __iter__ = interp2app(W_DictViewKeysObject.descr_iter),
+    __contains__ = interp2app(W_DictViewKeysObject.descr_contains),
 
     __eq__ = interp2app(W_DictViewKeysObject.descr_eq),
     __ne__ = interp2app(W_DictViewKeysObject.descr_ne),
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -806,6 +806,8 @@
         assert "a" in keys
         assert 10 not in keys
         assert "Z" not in keys
+        raises(TypeError, "[] in keys")     # [] is unhashable
+        raises(TypeError, keys.__contains__, [])
         assert d.viewkeys() == d.viewkeys()
         e = {1: 11, "a": "def"}
         assert d.viewkeys() == e.viewkeys()
@@ -831,6 +833,8 @@
         assert () not in items
         assert (1,) not in items
         assert (1, 2, 3) not in items
+        assert ([], []) not in items     # [] is unhashable, but no TypeError
+        assert not items.__contains__(([], []))
         assert d.viewitems() == d.viewitems()
         e = d.copy()
         assert d.viewitems() == e.viewitems()

From pypy.commits at gmail.com  Mon Feb 20 04:51:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 01:51:36 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58aabc28.5d532e0a.3978e.460b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90220:ae9e485aec01
Date: 2017-02-20 10:50 +0100
http://bitbucket.org/pypy/pypy/changeset/ae9e485aec01/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -159,3 +159,7 @@
 and ``space.newtext()``: these two methods are identical on PyPy 2.7 but
 not on PyPy 3.x.  The latter is used to get an app-level unicode string
 by decoding the RPython string, assumed to be utf-8.
+
+.. branch: fix_bool_restype
+
+Fix for ``ctypes.c_bool``-returning ctypes functions
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3835,7 +3835,7 @@
         assert (b > a) is False
         assert (b >= a) is True
         assert hash(a) == hash(b)
-    def assert_lt(a, b):
+    def assert_lt(a, b, check_hash=True):
         assert (a == b) is False
         assert (b == a) is False
         assert (a != b) is True
@@ -3848,9 +3848,10 @@
         assert (b <= a) is False
         assert (b > a) is True
         assert (b >= a) is True
-        assert hash(a) != hash(b)    # (or at least, it is unlikely)
-    def assert_gt(a, b):
-        assert_lt(b, a)
+        if check_hash:
+            assert hash(a) != hash(b)    # (or at least, it is unlikely)
+    def assert_gt(a, b, check_hash=True):
+        assert_lt(b, a, check_hash)
     def assert_ne(a, b):
         assert (a == b) is False
         assert (b == a) is False
@@ -3889,8 +3890,8 @@
     assert_eq(cast(t3, 65), 65)
     assert_ne(cast(t3, 65), b'A')
     assert_ne(cast(t3, 65), cast(t1, 65))
-    assert_gt(cast(t4, -1), -1)
-    assert_gt(cast(t4, -1), cast(t2, -1))
+    assert_gt(cast(t4, -1), -1, check_hash=False)
+    assert_gt(cast(t4, -1), cast(t2, -1), check_hash=False)
     assert_gt(cast(t4, -1), 99999)
     assert_eq(cast(t4, -1), 256 ** size_of_int() - 1)
     assert_eq(cast(t5, 3.0), 3)
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -26,7 +26,7 @@
 
 @slot_function([PyObject, Py_ssize_t], PyObject)
 def slot_sq_item(space, w_obj, index):
-    return space.getitem(w_obj, space.wrap(index))
+    return space.getitem(w_obj, space.newint(index))
 
 @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
 def slot_tp_new(space, w_type, w_args, w_kwds):
@@ -39,12 +39,12 @@
     #     we know (since we are in this function) that self is not a cpytype
     from pypy.module.cpyext.typeobject import W_PyCTypeObject
     w_type0 = w_type
-    mro_w = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    mro_w = space.listview(space.getattr(w_type0, space.newtext('__mro__')))
     for w_m in mro_w[1:]:
         if not w_type0.is_cpytype():
             break
         w_type0 = w_m
-    w_impl = space.getattr(w_type0, space.wrap('__new__'))
+    w_impl = space.getattr(w_type0, space.newtext('__new__'))
     args = Arguments(space, [w_type],
                      w_stararg=w_args, w_starstararg=w_kwds)
     return space.call_args(w_impl, args)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1536,9 +1536,26 @@
     def descr_iter(self, space):
         return W_DictMultiIterItemsObject(space, self.w_dict.iteritems())
 
+    def descr_contains(self, space, w_item):
+        if not space.isinstance_w(w_item, space.w_tuple):
+            return space.w_False
+        try:
+            w_key, w_value = space.fixedview_unroll(w_item, 2)
+        except OperationError as e:
+            if e.async(space):
+                raise
+            w_found = None
+        else:
+            w_found = self.w_dict.getitem(w_key)
+        if w_found is None:
+            return space.w_False
+        return space.eq(w_value, w_found)
+
 class W_DictViewKeysObject(W_DictViewObject, SetLikeDictView):
     def descr_iter(self, space):
         return W_DictMultiIterKeysObject(space, self.w_dict.iterkeys())
+    def descr_contains(self, space, w_key):
+        return self.w_dict.descr_contains(space, w_key)
 
 class W_DictViewValuesObject(W_DictViewObject):
     def descr_iter(self, space):
@@ -1549,6 +1566,7 @@
     __repr__ = interp2app(W_DictViewItemsObject.descr_repr),
     __len__ = interp2app(W_DictViewItemsObject.descr_len),
     __iter__ = interp2app(W_DictViewItemsObject.descr_iter),
+    __contains__ = interp2app(W_DictViewItemsObject.descr_contains),
 
     __eq__ = interp2app(W_DictViewItemsObject.descr_eq),
     __ne__ = interp2app(W_DictViewItemsObject.descr_ne),
@@ -1573,6 +1591,7 @@
     __repr__ = interp2app(W_DictViewKeysObject.descr_repr),
     __len__ = interp2app(W_DictViewKeysObject.descr_len),
     __iter__ = interp2app(W_DictViewKeysObject.descr_iter),
+    __contains__ = interp2app(W_DictViewKeysObject.descr_contains),
 
     __eq__ = interp2app(W_DictViewKeysObject.descr_eq),
     __ne__ = interp2app(W_DictViewKeysObject.descr_ne),
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -778,6 +778,8 @@
         assert "a" in keys
         assert 10 not in keys
         assert "Z" not in keys
+        raises(TypeError, "[] in keys")     # [] is unhashable
+        raises(TypeError, keys.__contains__, [])
         assert d.keys() == d.keys()
         e = {1: 11, "a": "def"}
         assert d.keys() == e.keys()
@@ -803,6 +805,8 @@
         assert () not in items
         assert (1,) not in items
         assert (1, 2, 3) not in items
+        raises(TypeError, "([], []) not in items")     # [] is unhashable
+        raises(TypeError, items.__contains__, ([], []))
         assert d.items() == d.items()
         e = d.copy()
         assert d.items() == e.items()
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -698,8 +698,18 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for unicode" % (enc,))
+        if enc == 'utf-8':
+            from rpython.rlib import runicode
+            bookkeeper = getbookkeeper()
+            s_func = bookkeeper.immutablevalue(
+                             runicode.unicode_encode_utf_8_elidable)
+            s_errors = bookkeeper.immutablevalue('strict')
+            s_errorhandler = bookkeeper.immutablevalue(None)
+            s_allow_surr = bookkeeper.immutablevalue(True)
+            args = [self, self.len(), s_errors, s_errorhandler, s_allow_surr]
+            bookkeeper.emulate_pbc_call(bookkeeper.position_key, s_func, args)
         return SomeString(no_nul=self.no_nul)
-    method_encode.can_only_throw = []
+    method_encode.can_only_throw = [UnicodeEncodeError]
 
 
 class __extend__(SomeString):
@@ -731,6 +741,19 @@
         enc = s_enc.const
         if enc not in ('ascii', 'latin-1', 'utf-8'):
             raise AnnotatorError("Encoding %s not supported for strings" % (enc,))
+        if enc == 'utf-8':
+            from rpython.rlib import runicode
+            bookkeeper = getbookkeeper()
+            s_func = bookkeeper.immutablevalue(
+                            runicode.str_decode_utf_8_elidable)
+            s_errors = bookkeeper.immutablevalue('strict')
+            s_final = bookkeeper.immutablevalue(True)
+            s_errorhandler = bookkeeper.immutablevalue(
+                                    runicode.default_unicode_error_decode)
+            s_allow_surr = bookkeeper.immutablevalue(True)
+            args = [self, self.len(), s_errors, s_final, s_errorhandler,
+                    s_allow_surr]
+            bookkeeper.emulate_pbc_call(bookkeeper.position_key, s_func, args)
         return SomeUnicodeString(no_nul=self.no_nul)
     method_decode.can_only_throw = [UnicodeDecodeError]
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -6,7 +6,6 @@
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import jit
-from rpython.rlib.nonconst import NonConstant
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -107,9 +106,6 @@
         return u'', None, endingpos
     raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
-def ll_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos):
-    raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
-
 # ____________________________________________________________
 # utf-8
 
@@ -131,15 +127,6 @@
                      errorhandler=None, allow_surrogates=allow_surrogate_by_default):
     if errorhandler is None:
         errorhandler = default_unicode_error_decode
-    # NB. a bit messy because rtyper/rstr.py also calls the same
-    # function.  Make sure we annotate for the args it passes, too
-    if NonConstant(False):
-        s = NonConstant('?????')
-        size = NonConstant(12345)
-        errors = NonConstant('strict')
-        final = NonConstant(True)
-        errorhandler = ll_unicode_error_decode
-        allow_surrogates = NonConstant(True)
     return str_decode_utf_8_elidable(s, size, errors, final, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
@@ -348,14 +335,6 @@
     #
     if errorhandler is None:
         errorhandler = default_unicode_error_encode
-    # NB. a bit messy because rtyper/rstr.py also calls the same
-    # function.  Make sure we annotate for the args it passes, too
-    if NonConstant(False):
-        s = NonConstant(u'?????')
-        size = NonConstant(12345)
-        errors = NonConstant('strict')
-        # no errorhandler needed for rtyper/rstr.py
-        allow_surrogates = NonConstant(True)
     return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
                                          allow_surrogates=allow_surrogates)
 
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -13,17 +13,14 @@
 
 class AbstractStringRepr(Repr):
 
-    def __init__(self, *args):
-        Repr.__init__(self, *args)
-        self.rstr_decode_utf_8 = None
-
     @jit.elidable
     def ll_decode_utf8(self, llvalue):
         from rpython.rtyper.annlowlevel import hlstr
         from rpython.rlib import runicode
         value = hlstr(llvalue)
         assert value is not None
-        errorhandler = runicode.ll_unicode_error_decode
+        errorhandler = runicode.default_unicode_error_decode
+        # NB. keep the arguments in sync with annotator/unaryop.py
         u, pos = runicode.str_decode_utf_8_elidable(
             value, len(value), 'strict', True, errorhandler, True)
         # XXX maybe the whole ''.decode('utf-8') should be not RPython.
@@ -374,10 +371,6 @@
 
 class AbstractUnicodeRepr(AbstractStringRepr):
 
-    def __init__(self, *args):
-        AbstractStringRepr.__init__(self, *args)
-        self.runicode_encode_utf_8 = None
-
     def rtype_method_upper(self, hop):
         raise TyperError("Cannot do toupper on unicode string")
 
@@ -390,6 +383,7 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
+        # NB. keep the arguments in sync with annotator/unaryop.py
         bytes = runicode.unicode_encode_utf_8_elidable(
             s, len(s), 'strict', None, True)
         return self.ll.llstr(bytes)

From pypy.commits at gmail.com  Mon Feb 20 05:03:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 02:03:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: line number changed
Message-ID: <58aabed5.4b1b190a.22bd1.430d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90221:803933b393b9
Date: 2017-02-20 11:02 +0100
http://bitbucket.org/pypy/pypy/changeset/803933b393b9/

Log:	line number changed

diff --git a/lib-python/3/test/test_doctest.py b/lib-python/3/test/test_doctest.py
--- a/lib-python/3/test/test_doctest.py
+++ b/lib-python/3/test/test_doctest.py
@@ -436,7 +436,7 @@
     >>> tests = finder.find(sample_func)
 
     >>> print(tests)  # doctest: +ELLIPSIS
-    [<DocTest sample_func from ...:19 (1 example)>]
+    [<DocTest sample_func from ...:20 (1 example)>]
 
 The exact name depends on how test_doctest was invoked, so allow for
 leading path components.

From pypy.commits at gmail.com  Mon Feb 20 05:15:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 02:15:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix error message
Message-ID: <58aac1b7.090d2e0a.72101.4856@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90222:549b113c75b6
Date: 2017-02-20 11:14 +0100
http://bitbucket.org/pypy/pypy/changeset/549b113c75b6/

Log:	fix error message

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -72,7 +72,7 @@
 
     def fmt_X(self, w_value):
         "HEX formatting"
-        r = hex_num_helper(self.space, w_value)
+        r = hex_num_helper(self.space, w_value, fmt_for_error='%X')
         if self.f_alt:
             prefix = '0X'
         else:
@@ -618,14 +618,14 @@
 
 def format_num_helper_generator(fmt, digits, decoder=maybe_int,
                                 expect_text="a number"):
-    def format_num_helper(space, w_value):
+    def format_num_helper(space, w_value, fmt_for_error=fmt):
         if not space.isinstance_w(w_value, space.w_int):
             try:
                 w_value = decoder(space, w_value)
             except OperationError:
                 raise oefmt(space.w_TypeError,
                             "%s format: %s is required, not %T",
-                            fmt, expect_text, w_value)
+                            fmt_for_error, expect_text, w_value)
         try:
             value = space.int_w(w_value)
             return fmt % (value,)
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -82,8 +82,10 @@
         assert '-23' == '%d' % f
         assert '-23' == '%i' % f
         assert '-23' == '%u' % f
-        raises(TypeError, "'%x' % f")
-        raises(TypeError, "'%X' % f")
+        e = raises(TypeError, "'%x' % f")
+        assert str(e.value).startswith("%x format:")
+        e = raises(TypeError, "'%X' % f")
+        assert str(e.value).startswith("%X format:")
         raises(TypeError, "'%o' % f")
         assert '-23.456' == '%s' % f
         # for 'r' use a float that has an exact decimal rep:

From pypy.commits at gmail.com  Mon Feb 20 05:19:31 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Feb 2017 02:19:31 -0800 (PST)
Subject: [pypy-commit] pypy vmprof-native: step once more to discard signal
 frame
Message-ID: <58aac2b3.5c582e0a.c975c.3d97@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-native
Changeset: r90223:53e085c091e4
Date: 2017-02-13 17:30 +0100
http://bitbucket.org/pypy/pypy/changeset/53e085c091e4/

Log:	step once more to discard signal frame

diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -168,6 +168,7 @@
     while (signal) {
         int is_signal_frame = unw_is_signal_frame(&cursor);
         if (is_signal_frame) {
+            unw_step(&cursor); // step once more discard signal frame
             break;
         }
         int err = unw_step(&cursor);

From pypy.commits at gmail.com  Mon Feb 20 05:19:33 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Feb 2017 02:19:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: update to cryptography 1.7.2
Message-ID: <58aac2b5.6517190a.235d.46d3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90224:c6383e02884f
Date: 2017-02-20 11:18 +0100
http://bitbucket.org/pypy/pypy/changeset/c6383e02884f/

Log:	update to cryptography 1.7.2

diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -17,4 +17,4 @@
 
 # Crpytography version
 
-`c8f47ad2122efdd5e772aee13ed5d4c64e7d6086`
+Copied over release version `1.7.2`
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
@@ -10,6 +10,7 @@
 
 TYPES = """
 static const int Cryptography_HAS_AES_WRAP;
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT;
 
 struct aes_key_st {
     ...;
@@ -28,8 +29,22 @@
 """
 
 MACROS = """
+/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for
+   this in 1.0.1+. */
+void AES_ctr128_encrypt(const unsigned char *, unsigned char *,
+                        size_t, const AES_KEY *, unsigned char[],
+                        unsigned char[], unsigned int *);
 """
 
 CUSTOMIZATIONS = """
 static const long Cryptography_HAS_AES_WRAP = 1;
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0;
+void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *,
+                           size_t, const AES_KEY *,
+                           unsigned char[], unsigned char[],
+                           unsigned int *) = NULL;
+#else
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1;
+#endif
 """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cmac.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cmac.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cmac.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cmac.py
@@ -5,7 +5,7 @@
 from __future__ import absolute_import, division, print_function
 
 INCLUDES = """
-#if !defined(OPENSSL_NO_CMAC)
+#if !defined(OPENSSL_NO_CMAC) && CRYPTOGRAPHY_OPENSSL_101_OR_GREATER
 #include <openssl/cmac.h>
 #endif
 """
@@ -28,7 +28,7 @@
 """
 
 CUSTOMIZATIONS = """
-#if !defined(OPENSSL_NO_CMAC)
+#if !defined(OPENSSL_NO_CMAC) && CRYPTOGRAPHY_OPENSSL_101_OR_GREATER
 static const long Cryptography_HAS_CMAC = 1;
 #else
 static const long Cryptography_HAS_CMAC = 0;
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py
@@ -17,6 +17,8 @@
 #include <windows.h>
 #endif
 
+#define CRYPTOGRAPHY_OPENSSL_101_OR_GREATER \
+    (OPENSSL_VERSION_NUMBER >= 0x10001000)
 #define CRYPTOGRAPHY_OPENSSL_102_OR_GREATER \
     (OPENSSL_VERSION_NUMBER >= 0x10002000)
 #define CRYPTOGRAPHY_OPENSSL_102BETA2_OR_GREATER \
@@ -24,6 +26,8 @@
 #define CRYPTOGRAPHY_OPENSSL_110_OR_GREATER \
     (OPENSSL_VERSION_NUMBER >= 0x10100000)
 
+#define CRYPTOGRAPHY_OPENSSL_LESS_THAN_101 \
+    (OPENSSL_VERSION_NUMBER < 0x10001000)
 #define CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 \
     (OPENSSL_VERSION_NUMBER < 0x10002000)
 #define CRYPTOGRAPHY_OPENSSL_LESS_THAN_102BETA3 \
@@ -47,8 +51,12 @@
 """
 
 TYPES = """
+static const int CRYPTOGRAPHY_OPENSSL_101_OR_GREATER;
+
 static const int CRYPTOGRAPHY_OPENSSL_110_OR_GREATER;
 
+static const int CRYPTOGRAPHY_OPENSSL_LESS_THAN_101;
+
 static const int CRYPTOGRAPHY_OPENSSL_LESS_THAN_102I;
 
 static const int CRYPTOGRAPHY_IS_LIBRESSL;
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ec.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ec.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ec.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ec.py
@@ -14,6 +14,7 @@
 
 TYPES = """
 static const int Cryptography_HAS_EC;
+static const int Cryptography_HAS_EC_1_0_1;
 static const int Cryptography_HAS_EC2M;
 static const int Cryptography_HAS_EC_1_0_2;
 
@@ -326,6 +327,13 @@
 
 int (*EC_METHOD_get_field_type)(const EC_METHOD *) = NULL;
 
+#else
+static const long Cryptography_HAS_EC = 1;
+#endif
+
+#if defined(OPENSSL_NO_EC) || CRYPTOGRAPHY_OPENSSL_LESS_THAN_101
+static const long Cryptography_HAS_EC_1_0_1 = 0;
+
 int (*EC_KEY_get_flags)(const EC_KEY *) = NULL;
 void (*EC_KEY_set_flags)(EC_KEY *, int) = NULL;
 void (*EC_KEY_clear_flags)(EC_KEY *, int) = NULL;
@@ -333,9 +341,10 @@
 int (*EC_KEY_set_public_key_affine_coordinates)(
     EC_KEY *, BIGNUM *, BIGNUM *) = NULL;
 #else
-static const long Cryptography_HAS_EC = 1;
+static const long Cryptography_HAS_EC_1_0_1 = 1;
 #endif
 
+
 #if defined(OPENSSL_NO_EC) || defined(OPENSSL_NO_EC2M)
 static const long Cryptography_HAS_EC2M = 0;
 
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/rsa.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/rsa.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/rsa.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/rsa.py
@@ -20,6 +20,7 @@
 static const int RSA_F4;
 
 static const int Cryptography_HAS_PSS_PADDING;
+static const int Cryptography_HAS_MGF1_MD;
 static const int Cryptography_HAS_RSA_OAEP_MD;
 """
 
@@ -72,6 +73,12 @@
 CUSTOMIZATIONS = """
 static const long Cryptography_HAS_PSS_PADDING = 1;
 
+#if CRYPTOGRAPHY_OPENSSL_101_OR_GREATER
+static const long Cryptography_HAS_MGF1_MD = 1;
+#else
+static const long Cryptography_HAS_MGF1_MD = 0;
+int (*EVP_PKEY_CTX_set_rsa_mgf1_md)(EVP_PKEY_CTX *, EVP_MD *) = NULL;
+#endif
 #if defined(EVP_PKEY_CTX_set_rsa_oaep_md)
 static const long Cryptography_HAS_RSA_OAEP_MD = 1;
 #else
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -25,6 +25,7 @@
 static const long Cryptography_HAS_GET_SERVER_TMP_KEY;
 static const long Cryptography_HAS_SSL_CTX_SET_CLIENT_CERT_ENGINE;
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS;
+static const long Cryptography_HAS_NPN_NEGOTIATED;
 
 /* Internally invented symbol to tell us if SNI is supported */
 static const long Cryptography_HAS_TLSEXT_HOSTNAME;
@@ -43,8 +44,8 @@
 static const long Cryptography_HAS_SSL_SET_SSL_CTX;
 static const long Cryptography_HAS_SSL_OP_NO_TICKET;
 static const long Cryptography_HAS_NETBSD_D1_METH;
+static const long Cryptography_HAS_NEXTPROTONEG;
 static const long Cryptography_HAS_ALPN;
-static const long Cryptography_HAS_NEXTPROTONEG;
 static const long Cryptography_HAS_SET_CERT_CB;
 
 static const long SSL_FILETYPE_PEM;
@@ -362,6 +363,9 @@
 
 long SSL_session_reused(SSL *);
 
+/* NPN APIs were introduced in OpenSSL 1.0.1.  To continue to support earlier
+ * versions some special handling of these is necessary.
+ */
 void SSL_CTX_set_next_protos_advertised_cb(SSL_CTX *,
                                            int (*)(SSL *,
                                                    const unsigned char **,
@@ -410,7 +414,7 @@
 
 /* Added in 1.0.2 */
 const SSL_METHOD *SSL_CTX_get_ssl_method(SSL_CTX *);
-
+/* Added in 1.0.1 */
 int SSL_SESSION_set1_id_context(SSL_SESSION *, const unsigned char *,
                                 unsigned int);
 /* Added in 1.1.0 for the great opaquing of structs */
@@ -434,6 +438,28 @@
 """
 
 CUSTOMIZATIONS = """
+/* Added in 1.0.1 but we need it in all versions now due to the great
+   opaquing. */
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_101
+/* from ssl.h */
+#define SSL_F_SSL_SESSION_SET1_ID_CONTEXT 312
+#define SSL_R_SSL_SESSION_ID_CONTEXT_TOO_LONG 273
+/* from ssl/ssl_sess.c */
+int SSL_SESSION_set1_id_context(SSL_SESSION *s, const unsigned char *sid_ctx,
+                                unsigned int sid_ctx_len)
+{
+    if (sid_ctx_len > SSL_MAX_SID_CTX_LENGTH) {
+        SSLerr(SSL_F_SSL_SESSION_SET1_ID_CONTEXT,
+               SSL_R_SSL_SESSION_ID_CONTEXT_TOO_LONG);
+        return 0;
+    }
+    s->sid_ctx_length = sid_ctx_len;
+    memcpy(s->sid_ctx, sid_ctx, sid_ctx_len);
+
+    return 1;
+}
+#endif
+
 /* Added in 1.0.2 but we need it in all versions now due to the great
    opaquing. */
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 || defined(LIBRESSL_VERSION_NUMBER)
@@ -579,7 +605,36 @@
 static const long Cryptography_HAS_NETBSD_D1_METH = 1;
 #endif
 
+/* Because OPENSSL defines macros that claim lack of support for things, rather
+ * than macros that claim support for things, we need to do a version check in
+ * addition to a definition check. NPN was added in 1.0.1: for any version
+ * before that, there is no compatibility.
+ */
+#if defined(OPENSSL_NO_NEXTPROTONEG) || CRYPTOGRAPHY_OPENSSL_LESS_THAN_101
+static const long Cryptography_HAS_NEXTPROTONEG = 0;
+void (*SSL_CTX_set_next_protos_advertised_cb)(SSL_CTX *,
+                                              int (*)(SSL *,
+                                                      const unsigned char **,
+                                                      unsigned int *,
+                                                      void *),
+                                              void *) = NULL;
+void (*SSL_CTX_set_next_proto_select_cb)(SSL_CTX *,
+                                         int (*)(SSL *,
+                                                 unsigned char **,
+                                                 unsigned char *,
+                                                 const unsigned char *,
+                                                 unsigned int,
+                                                 void *),
+                                         void *) = NULL;
+int (*SSL_select_next_proto)(unsigned char **, unsigned char *,
+                             const unsigned char *, unsigned int,
+                             const unsigned char *, unsigned int) = NULL;
+void (*SSL_get0_next_proto_negotiated)(const SSL *,
+                                       const unsigned char **,
+                                       unsigned *) = NULL;
+#else
 static const long Cryptography_HAS_NEXTPROTONEG = 1;
+#endif
 
 /* ALPN was added in OpenSSL 1.0.2. */
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 && !defined(LIBRESSL_VERSION_NUMBER)
@@ -652,4 +707,13 @@
 static const long TLS_ST_BEFORE = 0;
 static const long TLS_ST_OK = 0;
 #endif
+
+/* This define is available in 1.0.1+ so we can remove this when we drop
+   support for 1.0.0 */
+#ifdef OPENSSL_NPN_NEGOTIATED
+static const long Cryptography_HAS_NPN_NEGOTIATED = 1;
+#else
+static const long OPENSSL_NPN_NEGOTIATED = -1;
+static const long Cryptography_HAS_NPN_NEGOTIATED = 0;
+#endif
 """

From pypy.commits at gmail.com  Mon Feb 20 06:02:52 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Feb 2017 03:02:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: consider GEN_RID while extracting the
 get_peer_alt_names (change in stdlib 3.5.3)
Message-ID: <58aaccdc.5591190a.86c8b.4c03@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90225:dd295fcef6cb
Date: 2017-02-20 12:02 +0100
http://bitbucket.org/pypy/pypy/changeset/dd295fcef6cb/

Log:	consider GEN_RID while extracting the get_peer_alt_names (change in
	stdlib 3.5.3)

diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -109,6 +109,19 @@
             elif _type == lib.GEN_URI:
                 v = _string_from_asn1(name.d.uniformResourceIdentifier)
                 peer_alt_names.append(("URI", v))
+            elif _type == lib.GEN_RID:
+                v = "Registered ID"
+                buf = ffi.new("char[2048]")
+
+                length = lib.i2t_ASN1_OBJECT(buf, 2047, name.d.rid);
+                if length < 0:
+                    # TODO _setSSLError(NULL, 0, __FILE__, __LINE__);
+                    raise NotImplementedError
+                elif length >= 2048:
+                    v = "<INVALID>"
+                else:
+                    v = _str_with_len(buf, length)
+                peer_alt_names.append(("Registered ID", v))
             else:
                 # for everything else, we use the OpenSSL print form
                 if _type not in (lib.GEN_OTHERNAME, lib.GEN_X400, \

From pypy.commits at gmail.com  Mon Feb 20 07:02:16 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 20 Feb 2017 04:02:16 -0800 (PST)
Subject: [pypy-commit] pypy default: two more wraps
Message-ID: <58aadac8.12c1190a.1b126.504c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r90226:5f26b57a7716
Date: 2017-02-20 12:16 +0100
http://bitbucket.org/pypy/pypy/changeset/5f26b57a7716/

Log:	two more wraps

diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -193,4 +193,4 @@
 
 def stack_almost_full(space):
     """Return True if the stack is more than 15/16th full."""
-    return space.wrap(rstack.stack_almost_full())
+    return space.newbool(rstack.stack_almost_full())
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -363,7 +363,7 @@
         # tuple is lost
         length = space.len_w(w_fromlist)
         for i in range(length):
-            w_name = space.getitem(w_fromlist, space.wrap(i))
+            w_name = space.getitem(w_fromlist, space.newint(i))
             if not space.isinstance_w(w_name, space.w_str):
                 raise oefmt(space.w_TypeError,
                     "'fromlist' items must be str, not %T", w_name)

From pypy.commits at gmail.com  Mon Feb 20 07:13:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 04:13:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip a test,
 make another pass by loosening the skip condition
Message-ID: <58aadd69.12142e0a.cb56f.4baf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90227:cdc6aac40ecf
Date: 2017-02-20 13:12 +0100
http://bitbucket.org/pypy/pypy/changeset/cdc6aac40ecf/

Log:	skip a test, make another pass by loosening the skip condition

diff --git a/lib-python/3/test/test_readline.py b/lib-python/3/test/test_readline.py
--- a/lib-python/3/test/test_readline.py
+++ b/lib-python/3/test/test_readline.py
@@ -111,7 +111,8 @@
 
 class TestReadline(unittest.TestCase):
 
-    @unittest.skipIf(readline._READLINE_VERSION < 0x0601 and not is_editline,
+    @unittest.skipIf(getattr(readline, '_READLINE_VERSION', 0x0601) < 0x0601
+                     and not is_editline,
                      "not supported in this library version")
     def test_init(self):
         # Issue #19884: Ensure that the ANSI sequence "\033[1034h" is not
@@ -121,6 +122,9 @@
                                               TERM='xterm-256color')
         self.assertEqual(stdout, b'')
 
+    @unittest.skipIf(not hasattr(readline,
+                                 'set_completion_display_matches_hook'),
+                     "function not reimplemented in pypy")
     def test_nonascii(self):
         try:
             readline.add_history("\xEB\xEF")

From pypy.commits at gmail.com  Mon Feb 20 07:43:51 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 20 Feb 2017 04:43:51 -0800 (PST)
Subject: [pypy-commit] pypy default: mark space.wrap @not_rpython
Message-ID: <58aae487.99012e0a.951b8.55dc@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r90228:7ce64de7ad35
Date: 2017-02-20 13:43 +0100
http://bitbucket.org/pypy/pypy/changeset/7ce64de7ad35/

Log:	mark space.wrap @not_rpython

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -133,10 +133,7 @@
         assert typedef is not None
         return self.fromcache(TypeCache).getorbuild(typedef)
 
-    # BACKCOMPAT: this function is still accepted for backward
-    # compatibility, but its usage should be progressively removed
-    # everywhere apart from tests.
-    #@not_rpython # only for tests
+    @not_rpython # only for tests
     @specialize.argtype(1)
     def wrap(self, x):
         """ Wraps the Python value 'x' into one of the wrapper classes. This

From pypy.commits at gmail.com  Mon Feb 20 09:37:38 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Feb 2017 06:37:38 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: openssl 1.1.x introduces i2t_ASN1_OBJECT,
 and the stdlib immedeatly uses it,
 instead of directly calling OBJ_obj2txt (an alias with 1 extra parameter)
Message-ID: <58aaff32.04162e0a.21a12.581a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90229:6ac694bd3fc5
Date: 2017-02-20 14:09 +0100
http://bitbucket.org/pypy/pypy/changeset/6ac694bd3fc5/

Log:	openssl 1.1.x introduces i2t_ASN1_OBJECT, and the stdlib immedeatly
	uses it, instead of directly calling OBJ_obj2txt (an alias with 1
	extra parameter)

diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -113,7 +113,7 @@
                 v = "Registered ID"
                 buf = ffi.new("char[2048]")
 
-                length = lib.i2t_ASN1_OBJECT(buf, 2047, name.d.rid);
+                length = lib.OBJ_obj2txt(buf, 2047, name.d.rid, 0)
                 if length < 0:
                     # TODO _setSSLError(NULL, 0, __FILE__, __LINE__);
                     raise NotImplementedError

From pypy.commits at gmail.com  Mon Feb 20 09:37:40 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Feb 2017 06:37:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: implement RAND_egd in pypy's _ssl module,
 `not bytearray([]) is True` (which seems to be the same rule as
 `not []`), fixes a bug in SSLSocket.read
Message-ID: <58aaff34.514d2e0a.6102d.5789@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90230:4a32251b3e93
Date: 2017-02-20 15:34 +0100
http://bitbucket.org/pypy/pypy/changeset/4a32251b3e93/

Log:	implement RAND_egd in pypy's _ssl module, `not bytearray([]) is
	True` (which seems to be the same rule as `not []`), fixes a bug in
	SSLSocket.read

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -434,8 +434,10 @@
 
         sock = self.get_socket_or_connection_gone()
 
-        if not buffer_into:
+        if buffer_into is None:
             dest = ffi.new("char[]", length)
+            if length == 0:
+                return b""
             mem = dest
         else:
             mem = ffi.from_buffer(buffer_into)
@@ -443,6 +445,8 @@
                 length = len(buffer_into)
                 if length > sys.maxsize:
                     raise OverflowError("maximum length can't fit in a C 'int'")
+                if len(buffer_into) == 0:
+                    return 0
 
         if sock:
             timeout = _socket_timeout(sock)
@@ -1450,3 +1454,11 @@
 
         return lib.SSL_TLSEXT_ERR_OK
 
+if lib.Cryptography_HAS_EGD:
+    def RAND_egd(path):
+        bytecount = lib.RAND_egd_bytes(ffi.from_buffer(path), len(path))
+        if bytecount == -1:
+            raise SSLError("EGD connection failed or EGD did not return "
+                           "enough data to seed the PRNG");
+        return bytecount
+
diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -133,7 +133,7 @@
                 v = _bio_get_str(biobuf)
                 idx = v.find(":")
                 if idx == -1:
-                    return None
+                    raise ValueError("Invalid value %s", v)
                 peer_alt_names.append((v[:idx], v[idx+1:]))
 
         free_func_addr = ffi.addressof(lib, "GENERAL_NAME_free")

From pypy.commits at gmail.com  Mon Feb 20 09:53:02 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 20 Feb 2017 06:53:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: memoryview is not pickleable,
 add test that fails
Message-ID: <58ab02ce.51072e0a.d002.e4c8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90231:2f6597ed3e91
Date: 2017-02-20 15:52 +0100
http://bitbucket.org/pypy/pypy/changeset/2f6597ed3e91/

Log:	memoryview is not pickleable, add test that fails

diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -471,6 +471,10 @@
         assert 'finished' in repr(y)
         assert y.gi_code is None
 
+    def test_memoryview(self):
+        import pickle
+        raises(TypeError, pickle.dumps, memoryview(b"abc"))
+
 class XAppTestGeneratorCloning:
 
     def setup_class(cls):
@@ -663,4 +667,3 @@
         assert tb.tb_next == result.tb_next
 
         restore_top_frame(tb.tb_frame, saved)
-

From pypy.commits at gmail.com  Mon Feb 20 10:18:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 07:18:49 -0800 (PST)
Subject: [pypy-commit] pypy default: add
 sysconfig.get_config_var('INCLUDEPY')
Message-ID: <58ab08d9.0b202e0a.93a0a.5bf9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90232:5c0c182a6a77
Date: 2017-02-20 16:18 +0100
http://bitbucket.org/pypy/pypy/changeset/5c0c182a6a77/

Log:	add sysconfig.get_config_var('INCLUDEPY')

diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,9 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break        
+                break
+        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+                                                 'include')
 
     if args:
         vals = []

From pypy.commits at gmail.com  Mon Feb 20 10:29:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 07:29:15 -0800 (PST)
Subject: [pypy-commit] pypy default: Trying a fix for OS/X,
 untested (john z. on pypy-dev)
Message-ID: <58ab0b4b.ca052e0a.2b677.4ccc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90233:31ecf3410703
Date: 2017-02-20 16:28 +0100
http://bitbucket.org/pypy/pypy/changeset/31ecf3410703/

Log:	Trying a fix for OS/X, untested (john z. on pypy-dev)

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -31,8 +31,12 @@
         return self.rpath_flags
 
     def _args_for_shared(self, args):
+        if hasattr(self, '_exe_name'):
+            target = os.path.basename(self._exe_name)
+        else:
+            target = '$(TARGET)'     # inside a Makefile
         return (list(self.shared_only)
-                + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup', '-flat_namespace']
+                + ['-dynamiclib', '-install_name', '@rpath/' + target, '-undefined', 'dynamic_lookup', '-flat_namespace']
                 + args)
 
     def _include_dirs_for_libffi(self):
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -51,6 +51,7 @@
         return ["-Wl,--export-dynamic"]
 
     def _link(self, cc, ofiles, link_args, standalone, exe_name):
+        self._exe_name = str(exe_name)
         args = [str(ofile) for ofile in ofiles] + link_args
         args += ['-o', str(exe_name)]
         if not standalone:

From pypy.commits at gmail.com  Mon Feb 20 11:51:13 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 20 Feb 2017 08:51:13 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: add myself
Message-ID: <58ab1e81.d7052e0a.891bc.5e1a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r5783:9249bdc0f90e
Date: 2017-02-20 17:51 +0100
http://bitbucket.org/pypy/extradoc/changeset/9249bdc0f90e/

Log:	add myself

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -15,6 +15,7 @@
 Manuel Jacob         25.02/04.03    Ermina
 Robert Zaremba       25.02/01.03    Ermina
 Ronan Lamy           25.02/04.03    Ermina
+Antonio Cuni         27.02/03.03    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Mon Feb 20 12:15:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 09:15:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: 'fix' the test by giving the behavior
 expected on PyPy, which is also (see docstring) more consistent imho.
Message-ID: <58ab2422.d9002e0a.25cf0.63b7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90234:9e34b1ddde5c
Date: 2017-02-20 18:14 +0100
http://bitbucket.org/pypy/pypy/changeset/9e34b1ddde5c/

Log:	'fix' the test by giving the behavior expected on PyPy, which is
	also (see docstring) more consistent imho.

diff --git a/lib-python/3/test/test_pdb.py b/lib-python/3/test/test_pdb.py
--- a/lib-python/3/test/test_pdb.py
+++ b/lib-python/3/test/test_pdb.py
@@ -914,6 +914,12 @@
 def test_pdb_issue_20766():
     """Test for reference leaks when the SIGINT handler is set.
 
+    Note a fix for PyPy: on CPython, the two iterations through the loop
+    don't stop at the same line each time.  Actually, if the loop
+    iterates more than twice, we have a behavior of period two(!).  This
+    is due to very internal behavior that could be classified as a bug
+    and that PyPy doesn't emulating exactly.
+
     >>> def test_function():
     ...     i = 1
     ...     while i <= 2:
@@ -929,8 +935,8 @@
     -> print('pdb %d: %s' % (i, sess._previous_sigint_handler))
     (Pdb) continue
     pdb 1: <built-in function default_int_handler>
-    > <doctest test.test_pdb.test_pdb_issue_20766[0]>(5)test_function()
-    -> sess.set_trace(sys._getframe())
+    > <doctest test.test_pdb.test_pdb_issue_20766[0]>(6)test_function()
+    -> print('pdb %d: %s' % (i, sess._previous_sigint_handler))
     (Pdb) continue
     pdb 2: <built-in function default_int_handler>
     """

From pypy.commits at gmail.com  Tue Feb 21 00:38:42 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Feb 2017 21:38:42 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix interaction of PyGILState_* with
 thread states
Message-ID: <58abd262.16502e0a.696c.6ced@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90235:03278118ee16
Date: 2017-02-21 05:38 +0000
http://bitbucket.org/pypy/pypy/changeset/03278118ee16/

Log:	Fix interaction of PyGILState_* with thread states

	Instead of nulling the ec.cpyext_threadstate when releasing the gil
	via PyEval_SaveThread(), we now keep it around and use a flag to
	emulate CPython's nulling of the thread state. This allows
	PyGILState_Ensure() to restore the thread state.

diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
-from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref
+    cpython_api, CANNOT_FAIL, cpython_struct)
+from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref
 from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rthread
@@ -29,8 +29,9 @@
     the current thread must have acquired it.  (This function is available even
     when thread support is disabled at compile time.)"""
     state = space.fromcache(InterpreterState)
-    tstate = state.swap_thread_state(
-        space, lltype.nullptr(PyThreadState.TO))
+    ec = space.getexecutioncontext()
+    tstate = state._get_thread_state(space, ec).memory
+    ec.cpyext_threadstate_is_current = False
     return tstate
 
 @cpython_api([PyThreadState], lltype.Void, gil="acquire")
@@ -40,8 +41,8 @@
     NULL.  If the lock has been created, the current thread must not have
     acquired it, otherwise deadlock ensues.  (This function is available even
     when thread support is disabled at compile time.)"""
-    state = space.fromcache(InterpreterState)
-    state.swap_thread_state(space, tstate)
+    ec = space.getexecutioncontext()
+    ec.cpyext_threadstate_is_current = True
 
 @cpython_api([], lltype.Void)
 def PyEval_InitThreads(space):
@@ -91,9 +92,11 @@
 # released, so a check against that can't be used to determine the need for
 # initialization).
 ExecutionContext.cpyext_initialized_threadstate = False
+ExecutionContext.cpyext_threadstate_is_current = True
 
 def cleanup_cpyext_state(self):
     self.cpyext_threadstate = None
+    self.cpyext_threadstate_is_current = True
     self.cpyext_initialized_threadstate = False
 ExecutionContext.cleanup_cpyext_state = cleanup_cpyext_state
 
@@ -162,6 +165,7 @@
         if not ec.cpyext_initialized_threadstate:
             ec.cpyext_threadstate = self.new_thread_state(space)
             ec.cpyext_initialized_threadstate = True
+            ec.cpyext_threadstate_is_current = True
         return ec.cpyext_threadstate
 
 @cpython_api([], PyThreadState, error=CANNOT_FAIL)
@@ -181,7 +185,7 @@
     meant that an exception was raised."""
     state = space.fromcache(InterpreterState)
     ts = state.get_thread_state(space)
-    if not ts:
+    if not space.getexecutioncontext().cpyext_threadstate_is_current:
         return lltype.nullptr(PyObject.TO)
     return ts.c_dict
 
@@ -189,8 +193,17 @@
 def PyThreadState_Swap(space, tstate):
     """Swap the current thread state with the thread state given by the argument
     tstate, which may be NULL.  The global interpreter lock must be held."""
+    ec = space.getexecutioncontext()
     state = space.fromcache(InterpreterState)
-    return state.swap_thread_state(space, tstate)
+    old_tstate = state.get_thread_state(space)
+    if not ec.cpyext_threadstate_is_current:
+        old_tstate = lltype.nullptr(PyThreadState.TO)
+    if tstate:
+        assert tstate == state.get_thread_state(space)
+        ec.cpyext_threadstate_is_current = True
+    else:
+        ec.cpyext_threadstate_is_current = False
+    return old_tstate
 
 @cpython_api([PyThreadState], lltype.Void, gil="acquire")
 def PyEval_AcquireThread(space, tstate):
@@ -254,6 +267,7 @@
         if not we_are_translated():
             _workaround_cpython_untranslated(space)
     #
+    ec.cpyext_threadstate_is_current = True
     return rffi.cast(PyGILState_STATE, previous_state)
 
 @cpython_api([PyGILState_STATE], lltype.Void, gil="pygilstate_release")
@@ -266,6 +280,7 @@
         assert ec.cpyext_gilstate_counter_noleave == 0
         assert oldstate == PyGILState_UNLOCKED
         space.threadlocals.leave_thread(space)
+    ec.cpyext_threadstate_is_current = False
 
 @cpython_api([], PyInterpreterState, error=CANNOT_FAIL)
 def PyInterpreterState_Head(space):
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -66,7 +66,7 @@
             raise
 
         try:
-            self.space.getexecutioncontext().cleanup_cpyext_threadstate()
+            self.space.getexecutioncontext().cleanup_cpyext_state()
         except AttributeError:
             pass
 
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -106,7 +106,6 @@
             del obj
         import gc; gc.collect()
 
-        space.getexecutioncontext().cleanup_cpyext_state()
 
         for w_obj in state.non_heaptypes_w:
             Py_DecRef(space, w_obj)
@@ -180,6 +179,7 @@
     def teardown_method(self, meth):
         if self.runappdirect:
             return
+        self.space.getexecutioncontext().cleanup_cpyext_state()
         self.cleanup_references(self.space)
         # XXX: like AppTestCpythonExtensionBase.teardown_method:
         # find out how to disable check_and_print_leaks() if the
@@ -370,6 +370,7 @@
             return
         for name in self.imported_module_names:
             self.unimport_module(name)
+        self.space.getexecutioncontext().cleanup_cpyext_state()
         self.cleanup_references(self.space)
         # XXX: find out how to disable check_and_print_leaks() if the
         # test failed...
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -69,8 +69,6 @@
         assert module.get() == 3
 
     def test_basic_threadstate_dance(self):
-        if self.runappdirect:
-            py.test.xfail('segfault: on cpython cannot Get() a NULL tstate')
         module = self.import_extension('foo', [
                 ("dance", "METH_NOARGS",
                  """
@@ -83,8 +81,8 @@
                          return PyLong_FromLong(0);
                      }
 
-                     new_tstate = PyThreadState_Get(); /* fails on cpython */
-                     if (new_tstate != NULL) {
+                     PyObject* d = PyThreadState_GetDict(); /* fails on cpython */
+                     if (d != NULL) {
                          return PyLong_FromLong(1);
                      }
 

From pypy.commits at gmail.com  Tue Feb 21 00:42:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 20 Feb 2017 21:42:26 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-threads-gil: Close branch
 cpyext-threads-gil.
Message-ID: <58abd342.d9002e0a.25cf0.7e3b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: cpyext-threads-gil
Changeset: r90236:c72b816f6e2a
Date: 2017-02-21 05:41 +0000
http://bitbucket.org/pypy/pypy/changeset/c72b816f6e2a/

Log:	Close branch cpyext-threads-gil.


From pypy.commits at gmail.com  Tue Feb 21 02:14:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 23:14:44 -0800 (PST)
Subject: [pypy-commit] pypy default: * only mark the tstate as released
 after we called PyGILState_Release()
Message-ID: <58abe8e4.0b8d190a.5cfd2.7bdd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90237:8e04fbcbc608
Date: 2017-02-21 08:14 +0100
http://bitbucket.org/pypy/pypy/changeset/8e04fbcbc608/

Log:	* only mark the tstate as released after we called
	PyGILState_Release() enough times to balance the number of calls to
	PyGILState_Ensure()

	* make potential errors print a descriptive message

diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -41,8 +41,7 @@
     NULL.  If the lock has been created, the current thread must not have
     acquired it, otherwise deadlock ensues.  (This function is available even
     when thread support is disabled at compile time.)"""
-    ec = space.getexecutioncontext()
-    ec.cpyext_threadstate_is_current = True
+    PyThreadState_Swap(space, tstate)
 
 @cpython_api([], lltype.Void)
 def PyEval_InitThreads(space):
@@ -199,7 +198,11 @@
     if not ec.cpyext_threadstate_is_current:
         old_tstate = lltype.nullptr(PyThreadState.TO)
     if tstate:
-        assert tstate == state.get_thread_state(space)
+        if tstate != state.get_thread_state(space):
+            print "Error in cpyext, CPython compatibility layer:"
+            print "PyThreadState_Swap() cannot be used to switch to another"
+            print "different PyThreadState right now"
+            raise AssertionError
         ec.cpyext_threadstate_is_current = True
     else:
         ec.cpyext_threadstate_is_current = False
@@ -279,8 +282,8 @@
     else:
         assert ec.cpyext_gilstate_counter_noleave == 0
         assert oldstate == PyGILState_UNLOCKED
+        ec.cpyext_threadstate_is_current = False
         space.threadlocals.leave_thread(space)
-    ec.cpyext_threadstate_is_current = False
 
 @cpython_api([], PyInterpreterState, error=CANNOT_FAIL)
 def PyInterpreterState_Head(space):

From pypy.commits at gmail.com  Tue Feb 21 02:16:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 23:16:34 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: fix dates from last I heard
Message-ID: <58abe952.11162e0a.87141.708b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5784:792034ae90a5
Date: 2017-02-21 08:16 +0100
http://bitbucket.org/pypy/extradoc/changeset/792034ae90a5/

Log:	fix dates from last I heard

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -13,9 +13,10 @@
 Richard Plangger     26.02/04.03    Ermina
 Remi Meier           (?)27.02/04.03 Ermina
 Manuel Jacob         25.02/04.03    Ermina
-Robert Zaremba       25.02/01.03    Ermina
+Robert Zaremba       26?.02/01.03   Ermina
 Ronan Lamy           25.02/04.03    Ermina
 Antonio Cuni         27.02/03.03    Ermina
+Maciej Fijalkowski   maybe          Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Tue Feb 21 02:38:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 23:38:27 -0800 (PST)
Subject: [pypy-commit] pypy default: A test that fails before 8e04fbcbc608
Message-ID: <58abee73.8c092e0a.cd48d.796a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90238:fcfa989b8c49
Date: 2017-02-21 08:37 +0100
http://bitbucket.org/pypy/pypy/changeset/fcfa989b8c49/

Log:	A test that fails before 8e04fbcbc608

diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -173,6 +173,26 @@
         res = module.bounce()
         assert res == 4
 
+    def test_nested_pygilstate_ensure(self):
+        module = self.import_extension('foo', [
+            ("bounce", "METH_NOARGS",
+            """
+            PyGILState_STATE gilstate;
+            PyThreadState *tstate;
+            PyObject *dict;
+
+            if (PyEval_ThreadsInitialized() == 0)
+                PyEval_InitThreads();
+            dict = PyThreadState_GetDict();
+            gilstate = PyGILState_Ensure();
+            PyGILState_Release(gilstate);
+            if (PyThreadState_GetDict() != dict)
+                return PyLong_FromLong(-2);
+            return PyLong_FromLong(4);
+            """)])
+        res = module.bounce()
+        assert res == 4
+
     def test_threadsinitialized(self):
         module = self.import_extension('foo', [
                 ("test", "METH_NOARGS",

From pypy.commits at gmail.com  Tue Feb 21 02:42:43 2017
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 20 Feb 2017 23:42:43 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: definitive travel dates
Message-ID: <58abef73.c4561c0a.a6cd4.20a8@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: extradoc
Changeset: r5785:5ada3a95dafb
Date: 2017-02-21 08:42 +0100
http://bitbucket.org/pypy/extradoc/changeset/5ada3a95dafb/

Log:	definitive travel dates

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -11,7 +11,7 @@
 ==================== ============== =======================
 Armin Rigo                          private
 Richard Plangger     26.02/04.03    Ermina
-Remi Meier           (?)27.02/04.03 Ermina
+Remi Meier           27.02/04.03    Ermina
 Manuel Jacob         25.02/04.03    Ermina
 Robert Zaremba       26?.02/01.03   Ermina
 Ronan Lamy           25.02/04.03    Ermina

From pypy.commits at gmail.com  Tue Feb 21 02:50:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Feb 2017 23:50:39 -0800 (PST)
Subject: [pypy-commit] pypy default: fix for 31ecf3410703 running some tests
Message-ID: <58abf14f.5d542e0a.c46c7.79bd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90239:071247c0f952
Date: 2017-02-21 08:45 +0100
http://bitbucket.org/pypy/pypy/changeset/071247c0f952/

Log:	fix for 31ecf3410703 running some tests

diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -51,11 +51,12 @@
         return ["-Wl,--export-dynamic"]
 
     def _link(self, cc, ofiles, link_args, standalone, exe_name):
-        self._exe_name = str(exe_name)
         args = [str(ofile) for ofile in ofiles] + link_args
         args += ['-o', str(exe_name)]
         if not standalone:
+            self._exe_name = str(exe_name)
             args = self._args_for_shared(args)
+            del self._exe_name      # remove, otherwise __eq__() fails
         self._execute_c_compiler(cc, args, exe_name,
                                  cwd=str(exe_name.dirpath()))
         return exe_name

From pypy.commits at gmail.com  Tue Feb 21 03:05:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 00:05:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58abf4cc.504f190a.716b1.81a6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90240:f9cd9cb3c6b5
Date: 2017-02-21 08:04 +0000
http://bitbucket.org/pypy/pypy/changeset/f9cd9cb3c6b5/

Log:	hg merge default

diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,9 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break        
+                break
+        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+                                                 'include')
 
     if args:
         vals = []
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.api import (
-    cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct)
-from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref
+    cpython_api, CANNOT_FAIL, cpython_struct)
+from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref
 from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rthread
@@ -29,8 +29,9 @@
     the current thread must have acquired it.  (This function is available even
     when thread support is disabled at compile time.)"""
     state = space.fromcache(InterpreterState)
-    tstate = state.swap_thread_state(
-        space, lltype.nullptr(PyThreadState.TO))
+    ec = space.getexecutioncontext()
+    tstate = state._get_thread_state(space, ec).memory
+    ec.cpyext_threadstate_is_current = False
     return tstate
 
 @cpython_api([PyThreadState], lltype.Void, gil="acquire")
@@ -40,8 +41,7 @@
     NULL.  If the lock has been created, the current thread must not have
     acquired it, otherwise deadlock ensues.  (This function is available even
     when thread support is disabled at compile time.)"""
-    state = space.fromcache(InterpreterState)
-    state.swap_thread_state(space, tstate)
+    PyThreadState_Swap(space, tstate)
 
 @cpython_api([], lltype.Void)
 def PyEval_InitThreads(space):
@@ -91,9 +91,11 @@
 # released, so a check against that can't be used to determine the need for
 # initialization).
 ExecutionContext.cpyext_initialized_threadstate = False
+ExecutionContext.cpyext_threadstate_is_current = True
 
 def cleanup_cpyext_state(self):
     self.cpyext_threadstate = None
+    self.cpyext_threadstate_is_current = True
     self.cpyext_initialized_threadstate = False
 ExecutionContext.cleanup_cpyext_state = cleanup_cpyext_state
 
@@ -162,6 +164,7 @@
         if not ec.cpyext_initialized_threadstate:
             ec.cpyext_threadstate = self.new_thread_state(space)
             ec.cpyext_initialized_threadstate = True
+            ec.cpyext_threadstate_is_current = True
         return ec.cpyext_threadstate
 
 @cpython_api([], PyThreadState, error=CANNOT_FAIL)
@@ -185,7 +188,7 @@
     meant that an exception was raised."""
     state = space.fromcache(InterpreterState)
     ts = state.get_thread_state(space)
-    if not ts:
+    if not space.getexecutioncontext().cpyext_threadstate_is_current:
         return lltype.nullptr(PyObject.TO)
     return ts.c_dict
 
@@ -193,8 +196,21 @@
 def PyThreadState_Swap(space, tstate):
     """Swap the current thread state with the thread state given by the argument
     tstate, which may be NULL.  The global interpreter lock must be held."""
+    ec = space.getexecutioncontext()
     state = space.fromcache(InterpreterState)
-    return state.swap_thread_state(space, tstate)
+    old_tstate = state.get_thread_state(space)
+    if not ec.cpyext_threadstate_is_current:
+        old_tstate = lltype.nullptr(PyThreadState.TO)
+    if tstate:
+        if tstate != state.get_thread_state(space):
+            print "Error in cpyext, CPython compatibility layer:"
+            print "PyThreadState_Swap() cannot be used to switch to another"
+            print "different PyThreadState right now"
+            raise AssertionError
+        ec.cpyext_threadstate_is_current = True
+    else:
+        ec.cpyext_threadstate_is_current = False
+    return old_tstate
 
 @cpython_api([PyThreadState], lltype.Void, gil="acquire")
 def PyEval_AcquireThread(space, tstate):
@@ -258,6 +274,7 @@
         if not we_are_translated():
             _workaround_cpython_untranslated(space)
     #
+    ec.cpyext_threadstate_is_current = True
     return rffi.cast(PyGILState_STATE, previous_state)
 
 @cpython_api([PyGILState_STATE], lltype.Void, gil="pygilstate_release")
@@ -269,6 +286,7 @@
     else:
         assert ec.cpyext_gilstate_counter_noleave == 0
         assert oldstate == PyGILState_UNLOCKED
+        ec.cpyext_threadstate_is_current = False
         space.threadlocals.leave_thread(space)
 
 @cpython_api([], PyInterpreterState, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -66,7 +66,7 @@
             raise
 
         try:
-            self.space.getexecutioncontext().cleanup_cpyext_threadstate()
+            self.space.getexecutioncontext().cleanup_cpyext_state()
         except AttributeError:
             pass
 
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -105,7 +105,6 @@
             del obj
         import gc; gc.collect()
 
-        space.getexecutioncontext().cleanup_cpyext_state()
 
         for w_obj in state.non_heaptypes_w:
             Py_DecRef(space, w_obj)
@@ -182,6 +181,7 @@
     def teardown_method(self, meth):
         if self.runappdirect:
             return
+        self.space.getexecutioncontext().cleanup_cpyext_state()
         self.cleanup_references(self.space)
         # XXX: like AppTestCpythonExtensionBase.teardown_method:
         # find out how to disable check_and_print_leaks() if the
@@ -372,6 +372,7 @@
             return
         for name in self.imported_module_names:
             self.unimport_module(name)
+        self.space.getexecutioncontext().cleanup_cpyext_state()
         self.cleanup_references(self.space)
         # XXX: find out how to disable check_and_print_leaks() if the
         # test failed...
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -79,8 +79,8 @@
                          return PyLong_FromLong(0);
                      }
 
-                     new_tstate = PyThreadState_Get(); /* fails on cpython */
-                     if (new_tstate != NULL) {
+                     PyObject* d = PyThreadState_GetDict(); /* fails on cpython */
+                     if (d != NULL) {
                          return PyLong_FromLong(1);
                      }
 
@@ -171,6 +171,26 @@
         res = module.bounce()
         assert res == 4
 
+    def test_nested_pygilstate_ensure(self):
+        module = self.import_extension('foo', [
+            ("bounce", "METH_NOARGS",
+            """
+            PyGILState_STATE gilstate;
+            PyThreadState *tstate;
+            PyObject *dict;
+
+            if (PyEval_ThreadsInitialized() == 0)
+                PyEval_InitThreads();
+            dict = PyThreadState_GetDict();
+            gilstate = PyGILState_Ensure();
+            PyGILState_Release(gilstate);
+            if (PyThreadState_GetDict() != dict)
+                return PyLong_FromLong(-2);
+            return PyLong_FromLong(4);
+            """)])
+        res = module.bounce()
+        assert res == 4
+
     def test_threadsinitialized(self):
         module = self.import_extension('foo', [
                 ("test", "METH_NOARGS",
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -135,10 +135,7 @@
         assert typedef is not None
         return self.fromcache(TypeCache).getorbuild(typedef)
 
-    # BACKCOMPAT: this function is still accepted for backward
-    # compatibility, but its usage should be progressively removed
-    # everywhere apart from tests.
-    #@not_rpython # only for tests
+    @not_rpython # only for tests
     @specialize.argtype(1)
     def wrap(self, x):
         """ Wraps the Python value 'x' into one of the wrapper classes. This
diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -31,8 +31,12 @@
         return self.rpath_flags
 
     def _args_for_shared(self, args):
+        if hasattr(self, '_exe_name'):
+            target = os.path.basename(self._exe_name)
+        else:
+            target = '$(TARGET)'     # inside a Makefile
         return (list(self.shared_only)
-                + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup', '-flat_namespace']
+                + ['-dynamiclib', '-install_name', '@rpath/' + target, '-undefined', 'dynamic_lookup', '-flat_namespace']
                 + args)
 
     def _include_dirs_for_libffi(self):
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -51,6 +51,7 @@
         return ["-Wl,--export-dynamic"]
 
     def _link(self, cc, ofiles, link_args, standalone, exe_name):
+        self._exe_name = str(exe_name)
         args = [str(ofile) for ofile in ofiles] + link_args
         args += ['-o', str(exe_name)]
         if not standalone:

From pypy.commits at gmail.com  Tue Feb 21 03:54:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 00:54:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Remove some space.wrap()
Message-ID: <58ac002c.c3d1190a.4dcd1.7411@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90241:04c87a175ddd
Date: 2017-02-21 09:33 +0100
http://bitbucket.org/pypy/pypy/changeset/04c87a175ddd/

Log:	Remove some space.wrap()

diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -31,7 +31,7 @@
             w_text = space.newunicode(text)
         if self.filename is not None:
             w_filename = space.wrap_fsdecoded(self.filename)
-        return space.newtuple([space.wrap(self.msg),
+        return space.newtuple([space.newtext(self.msg),
                                space.newtuple([w_filename,
                                                space.newint(self.lineno),
                                                space.newint(offset),
diff --git a/pypy/module/__pypy__/interp_stderrprinter.py b/pypy/module/__pypy__/interp_stderrprinter.py
--- a/pypy/module/__pypy__/interp_stderrprinter.py
+++ b/pypy/module/__pypy__/interp_stderrprinter.py
@@ -17,21 +17,21 @@
 
     def descr_repr(self, space):
         addrstring = unicode(self.getaddrstring(space))
-        return space.wrap(u"<StdErrPrinter(fd=%d) object at 0x%s>" %
-                          (self.fd, addrstring))
+        return space.newunicode(u"<StdErrPrinter(fd=%d) object at 0x%s>" %
+                                (self.fd, addrstring))
 
     def descr_noop(self, space):
         pass
 
     def descr_fileno(self, space):
-        return space.wrap(self.fd)
+        return space.newint(self.fd)
 
     def descr_isatty(self, space):
         try:
             res = os.isatty(self.fd)
         except OSError as e:
             raise wrap_oserror(space, e)
-        return space.wrap(res)
+        return space.newbool(res)
 
     def descr_write(self, space, w_data):
         # Encode to UTF-8-nosg.
@@ -43,16 +43,16 @@
             if e.errno == errno.EAGAIN:
                 return space.w_None
             raise wrap_oserror(space, e)
-        return space.wrap(n)
+        return space.newint(n)
 
     def descr_get_closed(self, space):
-        return space.wrap(False)
+        return space.newbool(False)
 
     def descr_get_encoding(self, space):
         return space.w_None
 
     def descr_get_mode(self, space):
-        return space.wrap(u'w')
+        return space.newtext('w')
 
 
 W_StdErrPrinter.typedef = TypeDef("StdErrPrinter",
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -419,7 +419,7 @@
         if fd < 0:
             raise oefmt(space.w_ValueError, "file has no OS file descriptor")
         fd = os.dup(fd)
-        mode = space.str_w(space.getattr(w_fileobj, space.wrap("mode")))
+        mode = space.text_w(space.getattr(w_fileobj, space.newtext("mode")))
         try:
             w_fileobj.cffi_fileobj = CffiFileObj(fd, mode)
         except OSError as e:
diff --git a/pypy/module/cpyext/dictproxyobject.py b/pypy/module/cpyext/dictproxyobject.py
--- a/pypy/module/cpyext/dictproxyobject.py
+++ b/pypy/module/cpyext/dictproxyobject.py
@@ -7,4 +7,4 @@
 
 @cpython_api([PyObject], PyObject)
 def PyDictProxy_New(space, w_dict):
-    return space.wrap(W_DictProxyObject(w_dict))
+    return W_DictProxyObject(w_dict)
diff --git a/pypy/module/cpyext/exception.py b/pypy/module/cpyext/exception.py
--- a/pypy/module/cpyext/exception.py
+++ b/pypy/module/cpyext/exception.py
@@ -15,7 +15,7 @@
     """Return the traceback associated with the exception as a new reference, as
     accessible from Python through __traceback__.  If there is no
     traceback associated, this returns NULL."""
-    w_tb = space.getattr(w_exc, space.wrap('__traceback__'))
+    w_tb = space.getattr(w_exc, space.newtext('__traceback__'))
     if space.is_none(w_tb):
         return None
     return w_tb
@@ -25,7 +25,7 @@
 def PyException_SetTraceback(space, w_exc, w_tb):
     """Set the traceback associated with the exception to tb.  Use Py_None to
     clear it."""
-    space.setattr(w_exc, space.wrap('__traceback__'), w_tb)
+    space.setattr(w_exc, space.newtext('__traceback__'), w_tb)
     return 0
 
 
@@ -35,7 +35,7 @@
     raised) associated with the exception as a new reference, as accessible from
     Python through __context__.  If there is no context associated, this
     returns NULL."""
-    w_ctx = space.getattr(w_exc, space.wrap('__context__'))
+    w_ctx = space.getattr(w_exc, space.newtext('__context__'))
     if space.is_none(w_ctx):
         return None
     return w_ctx
@@ -51,7 +51,7 @@
         Py_DecRef(space, ctx)
     else:
         w_ctx = space.w_None
-    space.setattr(w_exc, space.wrap('__context__'), w_ctx)
+    space.setattr(w_exc, space.newtext('__context__'), w_ctx)
 
 @cpython_api([PyObject], PyObject)
 def PyException_GetCause(space, w_exc):
@@ -59,7 +59,7 @@
     associated with the exception as a new reference, as accessible from Python
     through __cause__.  If there is no cause associated, this returns
     NULL."""
-    w_cause = space.getattr(w_exc, space.wrap('__cause__'))
+    w_cause = space.getattr(w_exc, space.newtext('__cause__'))
     if space.is_none(w_cause):
         return None
     return w_cause
@@ -75,5 +75,5 @@
         Py_DecRef(space, cause)
     else:
         w_cause = space.w_None
-    space.setattr(w_exc, space.wrap('__cause__'), w_cause)
+    space.setattr(w_exc, space.newtext('__cause__'), w_cause)
 
diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py
--- a/pypy/module/cpyext/structmember.py
+++ b/pypy/module/cpyext/structmember.py
@@ -121,7 +121,7 @@
             casted = rffi.cast(lltyp, value)
             if range_checking:
                 if rffi.cast(lltype.typeOf(value), casted) != value:
-                    space.warn(space.wrap("structmember: truncation of value"),
+                    space.warn(space.newtext("structmember: truncation of value"),
                                space.w_RuntimeWarning)
             array[0] = casted
             return 0
diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -58,7 +58,7 @@
             rposix_scandir.closedir(self.dirp)
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def fail(self, err=None):
         dirp = self.dirp
@@ -99,7 +99,7 @@
         finally:
             self._in_next = False
         direntry = W_DirEntry(self, name, known_type, inode)
-        return space.wrap(direntry)
+        return direntry
 
 
 W_ScandirIterator.typedef = TypeDef(
@@ -139,7 +139,7 @@
 
     def descr_repr(self, space):
         u = space.unicode_w(space.repr(self.w_name))
-        return space.wrap(u"<DirEntry %s>" % u)
+        return space.newunicode(u"<DirEntry %s>" % u)
 
     def fget_name(self, space):
         return self.w_name
@@ -272,16 +272,16 @@
     @unwrap_spec(follow_symlinks=bool)
     def descr_is_dir(self, space, __kwonly__, follow_symlinks=True):
         """return True if the entry is a directory; cached per entry"""
-        return space.wrap(self.is_dir(follow_symlinks))
+        return space.newbool(self.is_dir(follow_symlinks))
 
     @unwrap_spec(follow_symlinks=bool)
     def descr_is_file(self, space, __kwonly__, follow_symlinks=True):
         """return True if the entry is a file; cached per entry"""
-        return space.wrap(self.is_file(follow_symlinks))
+        return space.newbool(self.is_file(follow_symlinks))
 
     def descr_is_symlink(self, space):
         """return True if the entry is a symbolic link; cached per entry"""
-        return space.wrap(self.is_symlink())
+        return space.newbool(self.is_symlink())
 
     @unwrap_spec(follow_symlinks=bool)
     def descr_stat(self, space, __kwonly__, follow_symlinks=True):
@@ -294,7 +294,7 @@
         return build_stat_result(space, st)
 
     def descr_inode(self, space):
-        return space.wrap(self.inode)
+        return space.newint(self.inode)
 
 
 W_DirEntry.typedef = TypeDef(
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -394,7 +394,7 @@
         # the signal isn't a member of the mask or the signal was
         # invalid, and an invalid signal must have been our fault in
         # constructing the loop boundaries.
-        signals_w.append(space.wrap(sig))
+        signals_w.append(space.newint(sig))
     return space.call_function(space.w_set, space.newtuple(signals_w))
 
 def sigwait(space, w_signals):
@@ -404,7 +404,7 @@
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
             signum = signum_ptr[0]
-    return space.wrap(signum)
+    return space.newint(signum)
 
 def sigpending(space):
     with lltype.scoped_alloc(c_sigset_t.TO) as mask:
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -93,15 +93,15 @@
     HASH_WIDTH = 8 * rffi.sizeof(lltype.Signed)
     HASH_CUTOFF = 0
     info_w = [
-        space.wrap(HASH_WIDTH),
-        space.wrap(HASH_MODULUS),
-        space.wrap(HASH_INF),
-        space.wrap(HASH_NAN),
-        space.wrap(HASH_IMAG),
-        space.wrap(HASH_ALGORITHM),
-        space.wrap(HASH_HASH_BITS),
-        space.wrap(HASH_SEED_BITS),
-        space.wrap(HASH_CUTOFF),
+        space.newint(HASH_WIDTH),
+        space.newint(HASH_MODULUS),
+        space.newint(HASH_INF),
+        space.newint(HASH_NAN),
+        space.newint(HASH_IMAG),
+        space.newtext(HASH_ALGORITHM),
+        space.newint(HASH_HASH_BITS),
+        space.newint(HASH_SEED_BITS),
+        space.newint(HASH_CUTOFF),
     ]
     w_hash_info = app.wget(space, "hash_info")
     return space.call_function(w_hash_info, space.newtuple(info_w))
@@ -114,10 +114,10 @@
         return None
     from rpython.rlib import rthread
     if rthread.RPYTHREAD_NAME == "pthread":
-        w_lock = space.wrap("semaphore" if rthread.USE_SEMAPHORES
-                            else "mutex+cond")
+        w_lock = space.newtext("semaphore" if rthread.USE_SEMAPHORES
+                               else "mutex+cond")
         if rthread.CS_GNU_LIBPTHREAD_VERSION is not None:
-            w_version = space.wrap(
+            w_version = space.newtext(
                 os.confstr(rthread.CS_GNU_LIBPTHREAD_VERSION))
         else:
             w_version = space.w_None
@@ -125,7 +125,7 @@
         w_lock = space.w_None
         w_version = space.w_None
     info_w = [
-        space.wrap(rthread.RPYTHREAD_NAME),
+        space.newtext(rthread.RPYTHREAD_NAME),
         w_lock, w_version,
     ]
     w_thread_info = app.wget(space, "thread_info")
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -329,7 +329,7 @@
                     w_data = self.get_data(space, fname)
                     # XXX CPython does not handle the coding cookie either.
                     return space.call_method(w_data, "decode",
-                                             space.wrap("utf-8"))
+                                             space.newtext("utf-8"))
                 else:
                     found = True
         if found:
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -541,7 +541,7 @@
             if w_srctype is space.w_list or w_srctype is space.w_tuple:
                 length = space.len_w(w_source)
                 if jit.isconstant(length) and length == 1:
-                    w_item = space.getitem(w_source, space.wrap(0))
+                    w_item = space.getitem(w_source, space.newint(0))
                     value = getbytevalue(space, w_item)
                     return W_BytesObject(value)
             else:
@@ -578,7 +578,7 @@
 
     def descr_str(self, space):
         if space.sys.get_flag('bytes_warning'):
-            space.warn(space.wrap("str() on a bytes instance"),
+            space.warn(space.newtext("str() on a bytes instance"),
                        space.w_BytesWarning)
         return self.descr_repr(space)
 
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -78,7 +78,7 @@
         return space.newint(_hash_long(space, self.asbigint()))
 
     def descr_str(self, space):
-        return space.wrap(self.asbigint().str())
+        return space.newtext(self.asbigint().str())
     descr_repr = descr_str
 
 
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -226,9 +226,8 @@
                 w_arg = space.getitem(self.w_kwargs, w_kwarg)
             else:
                 if self.args is None:
-                    w_msg = space.wrap("Format string contains positional "
-                                       "fields")
-                    raise OperationError(space.w_ValueError, w_msg)
+                    raise oefmt(space.w_ValueError,
+                                "Format string contains positional fields")
                 try:
                     w_arg = self.args[index]
                 except IndexError:
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -147,7 +147,7 @@
             raise oefmt(space.w_IndexError, self._KIND1 + " index out of range")
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(self, W_BytesObject):
-            return space.wrap(ord(character))
+            return space.newint(ord(character))
         return self._new(character)
 
     def descr_capitalize(self, space):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -751,7 +751,7 @@
             continue
         msg = ("metaclass conflict: the metaclass of a derived class must be "
                "a (non-strict) subclass of the metaclasses of all its bases")
-        raise OperationError(space.w_TypeError, space.wrap(msg))
+        raise oefmt(space.w_TypeError, msg)
     return w_winner
 
 def _precheck_for_new(space, w_type):
@@ -796,7 +796,7 @@
 
 def descr_get__qualname__(space, w_type):
     w_type = _check(space, w_type)
-    return space.wrap(w_type.getqualname(space))
+    return space.newunicode(w_type.getqualname(space))
 
 def descr_set__qualname__(space, w_type, w_value):
     w_type = _check(space, w_type)

From pypy.commits at gmail.com  Tue Feb 21 03:54:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 00:54:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: more
Message-ID: <58ac002f.4b1b190a.22bd1.7e4d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90242:19201be06f79
Date: 2017-02-21 08:53 +0000
http://bitbucket.org/pypy/pypy/changeset/19201be06f79/

Log:	more

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -141,7 +141,7 @@
         from pypy.module.sys import system
         thread_info = system.get_thread_info(space)
         if thread_info is not None:
-            space.setitem(self.w_dict, space.wrap('thread_info'), thread_info)
+            space.setitem(self.w_dict, space.newtext('thread_info'), thread_info)
 
     def setup_after_space_initialization(self):
         "NOT_RPYTHON"
@@ -182,7 +182,7 @@
 
     def _file_is_closed(self, space, w_file):
         try:
-            w_closed = space.getattr(w_file, space.wrap('closed'))
+            w_closed = space.getattr(w_file, space.newtext('closed'))
         except OperationError:
             return False
         return space.bool_w(w_closed)

From pypy.commits at gmail.com  Tue Feb 21 04:25:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 01:25:23 -0800 (PST)
Subject: [pypy-commit] pypy default: unsure to understand why I need this to
 make py3.5 translate
Message-ID: <58ac0783.5d542e0a.c46c7.7ec9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90243:dd1a88425cf6
Date: 2017-02-21 09:23 +0000
http://bitbucket.org/pypy/pypy/changeset/dd1a88425cf6/

Log:	unsure to understand why I need this to make py3.5 translate

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -704,7 +704,8 @@
             s_func = bookkeeper.immutablevalue(
                              runicode.unicode_encode_utf_8_elidable)
             s_errors = bookkeeper.immutablevalue('strict')
-            s_errorhandler = bookkeeper.immutablevalue(None)
+            s_errorhandler = bookkeeper.immutablevalue(
+                                    runicode.default_unicode_error_encode)
             s_allow_surr = bookkeeper.immutablevalue(True)
             args = [self, self.len(), s_errors, s_errorhandler, s_allow_surr]
             bookkeeper.emulate_pbc_call(bookkeeper.position_key, s_func, args)
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -383,9 +383,10 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
+        errorhandler = runicode.default_unicode_error_encode
         # NB. keep the arguments in sync with annotator/unaryop.py
         bytes = runicode.unicode_encode_utf_8_elidable(
-            s, len(s), 'strict', None, True)
+            s, len(s), 'strict', errorhandler, True)
         return self.ll.llstr(bytes)
 
     def rtype_method_encode(self, hop):

From pypy.commits at gmail.com  Tue Feb 21 04:25:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 01:25:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58ac0785.e30f190a.19f1e.7e19@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90244:c91068b7e4ea
Date: 2017-02-21 09:24 +0000
http://bitbucket.org/pypy/pypy/changeset/c91068b7e4ea/

Log:	hg merge default

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -704,7 +704,8 @@
             s_func = bookkeeper.immutablevalue(
                              runicode.unicode_encode_utf_8_elidable)
             s_errors = bookkeeper.immutablevalue('strict')
-            s_errorhandler = bookkeeper.immutablevalue(None)
+            s_errorhandler = bookkeeper.immutablevalue(
+                                    runicode.default_unicode_error_encode)
             s_allow_surr = bookkeeper.immutablevalue(True)
             args = [self, self.len(), s_errors, s_errorhandler, s_allow_surr]
             bookkeeper.emulate_pbc_call(bookkeeper.position_key, s_func, args)
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -383,9 +383,10 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
+        errorhandler = runicode.default_unicode_error_encode
         # NB. keep the arguments in sync with annotator/unaryop.py
         bytes = runicode.unicode_encode_utf_8_elidable(
-            s, len(s), 'strict', None, True)
+            s, len(s), 'strict', errorhandler, True)
         return self.ll.llstr(bytes)
 
     def rtype_method_encode(self, hop):

From pypy.commits at gmail.com  Tue Feb 21 04:25:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 01:25:28 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58ac0788.4395190a.b50e4.858d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90245:fc792f1074fd
Date: 2017-02-21 09:24 +0000
http://bitbucket.org/pypy/pypy/changeset/fc792f1074fd/

Log:	merge heads

diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -51,11 +51,12 @@
         return ["-Wl,--export-dynamic"]
 
     def _link(self, cc, ofiles, link_args, standalone, exe_name):
-        self._exe_name = str(exe_name)
         args = [str(ofile) for ofile in ofiles] + link_args
         args += ['-o', str(exe_name)]
         if not standalone:
+            self._exe_name = str(exe_name)
             args = self._args_for_shared(args)
+            del self._exe_name      # remove, otherwise __eq__() fails
         self._execute_c_compiler(cc, args, exe_name,
                                  cwd=str(exe_name.dirpath()))
         return exe_name

From pypy.commits at gmail.com  Tue Feb 21 04:27:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 01:27:33 -0800 (PST)
Subject: [pypy-commit] pypy default: wrap -> newtext,
 but never reachable right now
Message-ID: <58ac0805.01462e0a.1ac84.7e7e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90246:3a0d83a1dcc2
Date: 2017-02-21 09:23 +0100
http://bitbucket.org/pypy/pypy/changeset/3a0d83a1dcc2/

Log:	wrap -> newtext, but never reachable right now

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -36,7 +36,7 @@
     try:
         cdll = capi.c_load_dictionary(name)
         if not cdll:
-           raise OperationError(space.w_RuntimeError, space.wrap(str("could not load dictionary " + name)))
+           raise OperationError(space.w_RuntimeError, space.newtext(str("could not load dictionary " + name)))
 
     except rdynload.DLOpenError as e:
         if hasattr(space, "fake"):      # FakeSpace fails e.msg (?!)

From pypy.commits at gmail.com  Tue Feb 21 04:27:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 01:27:35 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58ac0807.13a6190a.1b698.8072@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90247:c8b8ba5341a7
Date: 2017-02-21 10:26 +0100
http://bitbucket.org/pypy/pypy/changeset/c8b8ba5341a7/

Log:	merge heads

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -704,7 +704,8 @@
             s_func = bookkeeper.immutablevalue(
                              runicode.unicode_encode_utf_8_elidable)
             s_errors = bookkeeper.immutablevalue('strict')
-            s_errorhandler = bookkeeper.immutablevalue(None)
+            s_errorhandler = bookkeeper.immutablevalue(
+                                    runicode.default_unicode_error_encode)
             s_allow_surr = bookkeeper.immutablevalue(True)
             args = [self, self.len(), s_errors, s_errorhandler, s_allow_surr]
             bookkeeper.emulate_pbc_call(bookkeeper.position_key, s_func, args)
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -383,9 +383,10 @@
         from rpython.rlib import runicode
         s = hlunicode(ll_s)
         assert s is not None
+        errorhandler = runicode.default_unicode_error_encode
         # NB. keep the arguments in sync with annotator/unaryop.py
         bytes = runicode.unicode_encode_utf_8_elidable(
-            s, len(s), 'strict', None, True)
+            s, len(s), 'strict', errorhandler, True)
         return self.ll.llstr(bytes)
 
     def rtype_method_encode(self, hop):
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -51,11 +51,12 @@
         return ["-Wl,--export-dynamic"]
 
     def _link(self, cc, ofiles, link_args, standalone, exe_name):
-        self._exe_name = str(exe_name)
         args = [str(ofile) for ofile in ofiles] + link_args
         args += ['-o', str(exe_name)]
         if not standalone:
+            self._exe_name = str(exe_name)
             args = self._args_for_shared(args)
+            del self._exe_name      # remove, otherwise __eq__() fails
         self._execute_c_compiler(cc, args, exe_name,
                                  cwd=str(exe_name.dirpath()))
         return exe_name

From pypy.commits at gmail.com  Tue Feb 21 04:27:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 01:27:37 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58ac0809.99012e0a.951b8.8be2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90248:a56077f0591a
Date: 2017-02-21 10:26 +0100
http://bitbucket.org/pypy/pypy/changeset/a56077f0591a/

Log:	hg merge default

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -36,7 +36,7 @@
     try:
         cdll = capi.c_load_dictionary(name)
         if not cdll:
-           raise OperationError(space.w_RuntimeError, space.wrap(str("could not load dictionary " + name)))
+           raise OperationError(space.w_RuntimeError, space.newtext(str("could not load dictionary " + name)))
 
     except rdynload.DLOpenError as e:
         if hasattr(space, "fake"):      # FakeSpace fails e.msg (?!)
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -51,11 +51,12 @@
         return ["-Wl,--export-dynamic"]
 
     def _link(self, cc, ofiles, link_args, standalone, exe_name):
-        self._exe_name = str(exe_name)
         args = [str(ofile) for ofile in ofiles] + link_args
         args += ['-o', str(exe_name)]
         if not standalone:
+            self._exe_name = str(exe_name)
             args = self._args_for_shared(args)
+            del self._exe_name      # remove, otherwise __eq__() fails
         self._execute_c_compiler(cc, args, exe_name,
                                  cwd=str(exe_name.dirpath()))
         return exe_name

From pypy.commits at gmail.com  Tue Feb 21 05:18:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 02:18:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <58ac1406.12c1190a.1b126.8911@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90249:4ead37a14a0e
Date: 2017-02-21 11:18 +0100
http://bitbucket.org/pypy/pypy/changeset/4ead37a14a0e/

Log:	fix test

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -354,8 +354,8 @@
         o = __import__('sys', [], [], ['']) # CPython accepts this
         assert sys == o
 
-    def test_import_fromlist_must_not_contain_unicodes(self):
-        raises(TypeError, __import__, 'encodings', None, None, [u'xxx'])
+    def test_import_fromlist_must_not_contain_bytes(self):
+        raises(TypeError, __import__, 'encodings', None, None, [b'xxx'])
 
     def test_proper_failure_on_killed__path__(self):
         import pkg.pkg2.a

From pypy.commits at gmail.com  Tue Feb 21 05:22:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 02:22:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: missed lines in the merge 9690cd452a80
Message-ID: <58ac14d9.d9002e0a.25cf0.8d48@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90250:d08fd55e765a
Date: 2017-02-21 11:21 +0100
http://bitbucket.org/pypy/pypy/changeset/d08fd55e765a/

Log:	missed lines in the merge 9690cd452a80

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -153,6 +153,15 @@
         pass
     p.join('x.py').rename(p.join('x.pyw'))
 
+    if hasattr(p, "mksymlinkto"):
+        p = root.join("devnullpkg")
+        p.ensure(dir=True)
+        p.join("__init__.py").mksymlinkto(os.devnull)
+
+    p = root.join("onlypyw")
+    p.ensure(dir=True)
+    p.join("__init__.pyw")
+
     return str(root)
 
 

From pypy.commits at gmail.com  Tue Feb 21 05:26:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 02:26:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test_version.py
Message-ID: <58ac15f1.a9212e0a.f2500.8bc5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90251:edb4e5cd554c
Date: 2017-02-21 11:25 +0100
http://bitbucket.org/pypy/pypy/changeset/edb4e5cd554c/

Log:	fix test_version.py

diff --git a/pypy/module/cpyext/include/pyconfig.h b/pypy/module/cpyext/include/pyconfig.h
--- a/pypy/module/cpyext/include/pyconfig.h
+++ b/pypy/module/cpyext/include/pyconfig.h
@@ -38,9 +38,9 @@
       * taken care of by distutils.) */
 #        ifdef _DEBUG
 #            error("debug first with cpython")    
-#            pragma comment(lib,"python33.lib")
+#            pragma comment(lib,"python35.lib")
 #        else
-#            pragma comment(lib,"python33.lib")
+#            pragma comment(lib,"python35.lib")
 #        endif /* _DEBUG */
 #    endif
 #endif /* _MSC_VER */

From pypy.commits at gmail.com  Tue Feb 21 05:32:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 02:32:28 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: don't crash if we read less lines than
 co_firstlineno (it can show up when trying to get the source of some
 half-builtin function in pypy)
Message-ID: <58ac173c.0b202e0a.93a0a.8c1f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90252:db9d87141352
Date: 2017-02-21 11:31 +0100
http://bitbucket.org/pypy/pypy/changeset/db9d87141352/

Log:	don't crash if we read less lines than co_firstlineno (it can show
	up when trying to get the source of some half-builtin function in
	pypy)

diff --git a/lib-python/3/inspect.py b/lib-python/3/inspect.py
--- a/lib-python/3/inspect.py
+++ b/lib-python/3/inspect.py
@@ -797,7 +797,7 @@
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise OSError('could not find function definition')
-        lnum = object.co_firstlineno - 1
+        lnum = min(object.co_firstlineno, len(lines)) - 1
         pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         while lnum > 0:
             if pat.match(lines[lnum]): break

From pypy.commits at gmail.com  Tue Feb 21 06:17:07 2017
From: pypy.commits at gmail.com (fijal)
Date: Tue, 21 Feb 2017 03:17:07 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: Start fighting on a new branch
 with utf8 being the default storage
Message-ID: <58ac21b3.0e152e0a.20484.8a63@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90253:c9309766eac6
Date: 2017-02-21 12:17 +0100
http://bitbucket.org/pypy/pypy/changeset/c9309766eac6/

Log:	Start fighting on a new branch with utf8 being the default storage
	for W_Unicode object

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -57,6 +57,7 @@
             assert 0 <= ps <= q
             substr = s[ps:q]
         else:
+            xxx
             substr = decode_unicode_utf8(space, s, ps, q)
         if rawmode:
             v = unicodehelper.decode_raw_unicode_escape(space, substr)
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -38,14 +38,16 @@
 def decode_unicode_escape(space, string):
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
-    result, consumed = runicode.str_decode_unicode_escape(
+    # XXX pick better length, maybe
+    result, consumed = runicode.str_decode_utf8_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         unicodedata_handler=unicodedata_handler)
     return result
 
 def decode_raw_unicode_escape(space, string):
-    result, consumed = runicode.str_decode_raw_unicode_escape(
+    # XXX pick better length, maybe
+    result, consumed = runicode.str_decode_raw_utf8_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space))
     return result
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -5,7 +5,7 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from rpython.rlib.runicode import UNICHR
+from rpython.rlib.runicode import UNICHR, unichr_as_utf8
 from rpython.rlib.rfloat import isnan, isinf, round_double
 from rpython.rlib import rfloat
 import __builtin__
@@ -25,12 +25,12 @@
 @unwrap_spec(code=int)
 def unichr(space, code):
     "Return a Unicode string of one character with the given ordinal."
-    # XXX range checking!
+    # XXX this assumes unichr would be happy to return you surrogates
     try:
-        c = UNICHR(code)
+        s = unichr_as_utf8(code)
     except ValueError:
         raise oefmt(space.w_ValueError, "unichr() arg out of range")
-    return space.newunicode(c)
+    return space.newunicode(s)
 
 def len(space, w_obj):
     "len(object) -> integer\n\nReturn the number of items of a sequence or mapping."
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -1,11 +1,10 @@
 """The builtin str implementation"""
 
 from rpython.rlib import jit
-from rpython.rlib.jit import we_are_jitted
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin)
 from rpython.rlib.buffer import StringBuffer
-from rpython.rlib.rstring import StringBuilder, replace
+from rpython.rlib.rstring import StringBuilder
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
@@ -18,7 +17,7 @@
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.unicodeobject import (
     decode_object, unicode_from_encoded_object,
-    unicode_from_string, getdefaultencoding)
+    getdefaultencoding)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -346,10 +346,10 @@
             return self.w_None
         return self.newtext(s)
 
-    def newunicode(self, uni):
-        assert uni is not None
-        assert isinstance(uni, unicode)
-        return W_UnicodeObject(uni)
+    def newunicode(self, utf8s):
+        assert utf8s is not None
+        assert isinstance(utf8s, str)
+        return W_UnicodeObject(utf8s)
 
     def type(self, w_obj):
         jit.promote(w_obj.__class__)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -7,7 +7,8 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
-    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
+    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii,
+    check_ascii, AsciiCheckError)
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
@@ -23,21 +24,24 @@
 
 __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode',
            'encode_object', 'decode_object', 'unicode_from_object',
-           'unicode_from_string', 'unicode_to_decimal_w']
+           'utf8_from_string', 'unicode_to_decimal_w']
 
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
-    _immutable_fields_ = ['_value']
+    _immutable_fields_ = ['_utf8']
 
-    @enforceargs(uni=unicode)
-    def __init__(self, unistr):
-        assert isinstance(unistr, unicode)
-        self._value = unistr
+    @enforceargs(utf8str=str)
+    def __init__(self, utf8str, ucs4str=None):
+        assert isinstance(utf8str, str)
+        if ucs4str is not None:
+            assert isinstance(ucs4str, unicode)
+        self._utf8 = utf8str
+        self._ucs4 = ucs4str
 
     def __repr__(self):
         """representation for debugging purposes"""
-        return "%s(%r)" % (self.__class__.__name__, self._value)
+        return "%s(%r)" % (self.__class__.__name__, self._utf8)
 
     def unwrap(self, space):
         # for testing
@@ -79,8 +83,8 @@
     def str_w(self, space):
         return space.text_w(space.str(self))
 
-    def unicode_w(self, space):
-        return self._value
+    def utf8_w(self, space):
+        return self._utf8
 
     def readbuf_w(self, space):
         from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
@@ -117,7 +121,7 @@
     def _len(self):
         return len(self._value)
 
-    _val = unicode_w
+    _val = utf8_w
 
     @staticmethod
     def _use_rstr_ops(space, w_other):
@@ -128,9 +132,10 @@
     @staticmethod
     def _op_val(space, w_other, strict=None):
         if isinstance(w_other, W_UnicodeObject):
-            return w_other._value
+            return w_other._utf8
         if space.isinstance_w(w_other, space.w_bytes):
-            return unicode_from_string(space, w_other)._value
+            return utf8_from_string(space, w_other)._utf8
+        yyy
         if strict:
             raise oefmt(space.w_TypeError,
                 "%s arg must be None, unicode or str", strict)
@@ -235,7 +240,7 @@
         return encode_object(space, self, None, None)
 
     def descr_hash(self, space):
-        x = compute_hash(self._value)
+        x = compute_hash(self._utf8)
         return space.newint(x)
 
     def descr_eq(self, space, w_other):
@@ -564,17 +569,19 @@
     return unicode_from_encoded_object(space, w_res, None, "strict")
 
 
-def unicode_from_string(space, w_bytes):
+def utf8_from_string(space, w_bytes):
     # this is a performance and bootstrapping hack
     encoding = getdefaultencoding(space)
     if encoding != 'ascii':
+        xxx
         return unicode_from_encoded_object(space, w_bytes, encoding, "strict")
     s = space.bytes_w(w_bytes)
     try:
-        return W_UnicodeObject(s.decode("ascii"))
-    except UnicodeDecodeError:
+        check_ascii(s)
+    except AsciiCheckError:
         # raising UnicodeDecodeError is messy, "please crash for me"
         return unicode_from_encoded_object(space, w_bytes, "ascii", "strict")
+    return W_UnicodeObject(s)
 
 
 class UnicodeDocstrings:
@@ -1121,7 +1128,7 @@
     return [s for s in value]
 
 
-W_UnicodeObject.EMPTY = W_UnicodeObject(u'')
+W_UnicodeObject.EMPTY = W_UnicodeObject('')
 
 
 # Helper for converting int/long
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -43,6 +43,36 @@
             return ord(u[0])
         raise TypeError
 
+def unichr_as_utf8(code):
+    """ Encode code (numeric value) as utf8 encoded string
+    """
+    if code < 0:
+        raise ValueError
+    if code < 0x80:
+        # Encode ASCII
+        return chr(code)
+    if code < 0x0800:
+        # Encode Latin-1
+        return chr((0xc0 | (code >> 6))) + chr((0x80 | (code & 0x3f)))
+    if code < 0x10000:
+        return (chr((0xe0 | (code >> 12))) +
+                chr((0x80 | ((code >> 6) & 0x3f))) +
+                chr((0x80 | (code & 0x3f))))
+    if code < 0x10ffff:
+        return (chr((0xf0 | (code >> 18))) +
+                chr((0x80 | ((code >> 12) & 0x3f))) +
+                chr((0x80 | ((code >> 6) & 0x3f))) +
+                chr((0x80 | (code & 0x3f))))
+    raise ValueError
+
+class AsciiCheckError(Exception):
+    pass
+
+def check_ascii(s):
+    for i in range(0, len(s)):
+        if ord(s[i]) & 0x80:
+            raise AsciiCheckError
+        
 if MAXUNICODE > sys.maxunicode:
     # A version of unichr which allows codes outside the BMP
     # even on narrow unicode builds.
@@ -1377,6 +1407,129 @@
 
     return builder.build(), pos
 
+def str_decode_utf8_escape(s, size, errors, final=False,
+                              errorhandler=None,
+                              unicodedata_handler=None):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+
+    if size == 0:
+        return '', 0
+
+    builder = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            builder.append(ch)
+            pos += 1
+            continue
+
+        # - Escapes
+        pos += 1
+        if pos >= size:
+            message = "\\ at end of string"
+            res, pos = errorhandler(errors, "unicodeescape",
+                                    message, s, pos-1, size)
+            builder.append(res)
+            continue
+
+        ch = s[pos]
+        pos += 1
+        # \x escapes
+        if ch == '\n': pass
+        elif ch == '\\': builder.append('\\')
+        elif ch == '\'': builder.append('\'')
+        elif ch == '\"': builder.append('\"')
+        elif ch == 'b' : builder.append('\b')
+        elif ch == 'f' : builder.append('\f')
+        elif ch == 't' : builder.append('\t')
+        elif ch == 'n' : builder.append('\n')
+        elif ch == 'r' : builder.append('\r')
+        elif ch == 'v' : builder.append('\v')
+        elif ch == 'a' : builder.append('\a')
+        elif '0' <= ch <= '7':
+            xxx
+            x = ord(ch) - ord('0')
+            if pos < size:
+                ch = s[pos]
+                if '0' <= ch <= '7':
+                    pos += 1
+                    x = (x<<3) + ord(ch) - ord('0')
+                    if pos < size:
+                        ch = s[pos]
+                        if '0' <= ch <= '7':
+                            pos += 1
+                            x = (x<<3) + ord(ch) - ord('0')
+            builder.append(unichr(x))
+        # hex escapes
+        # \xXX
+        elif ch == 'x':
+            xxx
+            digits = 2
+            message = "truncated \\xXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \uXXXX
+        elif ch == 'u':
+            xxx
+            digits = 4
+            message = "truncated \\uXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        #  \UXXXXXXXX
+        elif ch == 'U':
+            xxx
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \N{name}
+        elif ch == 'N' and unicodedata_handler is not None:
+            xxx
+            message = "malformed \\N character escape"
+            look = pos
+
+            if look < size and s[look] == '{':
+                # look for the closing brace
+                while look < size and s[look] != '}':
+                    look += 1
+                if look < size and s[look] == '}':
+                    # found a name.  look it up in the unicode database
+                    message = "unknown Unicode character name"
+                    name = s[pos+1:look]
+                    code = unicodedata_handler.call(name)
+                    if code < 0:
+                        res, pos = errorhandler(errors, "unicodeescape",
+                                                message, s, pos-1, look+1)
+                        builder.append(res)
+                        continue
+                    pos = look + 1
+                    if code <= MAXUNICODE:
+                        builder.append(UNICHR(code))
+                    else:
+                        code -= 0x10000L
+                        builder.append(unichr(0xD800 + (code >> 10)))
+                        builder.append(unichr(0xDC00 + (code & 0x03FF)))
+                else:
+                    res, pos = errorhandler(errors, "unicodeescape",
+                                            message, s, pos-1, look+1)
+                    builder.append(res)
+            else:
+                res, pos = errorhandler(errors, "unicodeescape",
+                                        message, s, pos-1, look+1)
+                builder.append(res)
+        else:
+            builder.append('\\')
+            builder.append(ch)
+
+    return builder.build(), pos
+
 def make_unicode_escape_function(pass_printable=False, unicode_output=False,
                                  quotes=False, prefix=None):
     # Python3 has two similar escape functions: One to implement
@@ -1497,6 +1650,54 @@
 # ____________________________________________________________
 # Raw unicode escape
 
+def str_decode_raw_utf8_escape(s, size, errors, final=False,
+                               errorhandler=None):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+    if size == 0:
+        return '', 0
+    result = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            result.append(ch)
+            pos += 1
+            continue
+
+        # \u-escapes are only interpreted iff the number of leading
+        # backslashes is odd
+        bs = pos
+        while pos < size:
+            pos += 1
+            if pos == size or s[pos] != '\\':
+                break
+            result.append('\\')
+
+        # we have a backslash at the end of the string, stop here
+        if pos >= size:
+            result.append('\\')
+            break
+
+        if ((pos - bs) & 1 == 0 or
+            pos >= size or
+            (s[pos] != 'u' and s[pos] != 'U')):
+            result.append('\\')
+            result.append(s[pos])
+            pos += 1
+            continue
+
+        digits = 4 if s[pos] == 'u' else 8
+        message = "truncated \\uXXXX"
+        pos += 1
+        xxx # change hexescape to deal with utf8
+        pos = hexescape(result, s, pos, digits,
+                        "rawunicodeescape", errorhandler, message, errors)
+
+    return result.build(), pos
+
 def str_decode_raw_unicode_escape(s, size, errors, final=False,
                                   errorhandler=None):
     if errorhandler is None:

From pypy.commits at gmail.com  Tue Feb 21 07:10:20 2017
From: pypy.commits at gmail.com (fijal)
Date: Tue, 21 Feb 2017 04:10:20 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: fix the nomenclature and put a few
 stop gaps in places we know need fixing
Message-ID: <58ac2e2c.501d190a.fc809.8b96@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90254:29c62a133d25
Date: 2017-02-21 13:04 +0100
http://bitbucket.org/pypy/pypy/changeset/29c62a133d25/

Log:	fix the nomenclature and put a few stop gaps in places we know need
	fixing

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -261,7 +261,7 @@
     def str_w(self, space):
         self._typed_unwrap_error(space, "string")
 
-    def unicode_w(self, space):
+    def utf8_w(self, space):
         self._typed_unwrap_error(space, "unicode")
 
     def bytearray_list_of_chars_w(self, space):
@@ -1673,8 +1673,8 @@
             raise oefmt(self.w_TypeError, "argument must be a string")
         return self.bytes_w(w_obj)
 
-    def unicode_w(self, w_obj):
-        return w_obj.unicode_w(self)
+    def utf8_w(self, w_obj):
+        return w_obj.utf8_w(self)
 
     def unicode0_w(self, w_obj):
         "Like unicode_w, but rejects strings with NUL bytes."
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -16,7 +16,7 @@
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.unicodeobject import (
-    decode_object, unicode_from_encoded_object,
+    decode_object, utf8_from_encoded_object,
     getdefaultencoding)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -59,9 +59,9 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        s1 = space.unicode_w(self)
-        s2 = space.unicode_w(w_other)
-        if len(s2) > 1:
+        s1 = space.utf8_w(self)
+        s2 = space.utf8_w(w_other)
+        if len(s2) > 2:
             return s1 is s2
         else:            # strings of len <= 1 are unique-ified
             return s1 == s2
@@ -135,11 +135,10 @@
             return w_other._utf8
         if space.isinstance_w(w_other, space.w_bytes):
             return utf8_from_string(space, w_other)._utf8
-        yyy
         if strict:
             raise oefmt(space.w_TypeError,
                 "%s arg must be None, unicode or str", strict)
-        return unicode_from_encoded_object(
+        return utf8_from_encoded_object(
             space, w_other, None, "strict")._value
 
     def _chr(self, char):
@@ -509,6 +508,7 @@
         if encoding == 'ascii':
             # XXX error handling
             s = space.charbuf_w(w_obj)
+            xxx
             try:
                 u = fast_str_decode_ascii(s)
             except ValueError:
@@ -517,11 +517,13 @@
                     s, len(s), None, final=True, errorhandler=eh)[0]
             return space.newunicode(u)
         if encoding == 'utf-8':
+            yyy
             s = space.charbuf_w(w_obj)
             eh = unicodehelper.decode_error_handler(space)
             return space.newunicode(str_decode_utf_8(
                     s, len(s), None, final=True, errorhandler=eh,
                     allow_surrogates=True)[0])
+    xxx
     w_codecs = space.getbuiltinmodule("_codecs")
     w_decode = space.getattr(w_codecs, space.newtext("decode"))
     if errors is None:
@@ -532,7 +534,7 @@
     return w_retval
 
 
-def unicode_from_encoded_object(space, w_obj, encoding, errors):
+def utf8_from_encoded_object(space, w_obj, encoding, errors):
     # explicitly block bytearray on 2.7
     from .bytearrayobject import W_BytearrayObject
     if isinstance(w_obj, W_BytearrayObject):
@@ -580,7 +582,7 @@
         check_ascii(s)
     except AsciiCheckError:
         # raising UnicodeDecodeError is messy, "please crash for me"
-        return unicode_from_encoded_object(space, w_bytes, "ascii", "strict")
+        return utf8_from_encoded_object(space, w_bytes, "ascii", "strict")
     return W_UnicodeObject(s)
 
 

From pypy.commits at gmail.com  Tue Feb 21 07:10:22 2017
From: pypy.commits at gmail.com (fijal)
Date: Tue, 21 Feb 2017 04:10:22 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: shuffle stuff around to new file
 rutf8
Message-ID: <58ac2e2e.1a0e2e0a.4eca.8399@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90255:76c0add1d61e
Date: 2017-02-21 13:10 +0100
http://bitbucket.org/pypy/pypy/changeset/76c0add1d61e/

Log:	shuffle stuff around to new file rutf8

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib import runicode
+from rpython.rlib import runicode, rutf8
 from pypy.module._codecs import interp_codecs
 
 @specialize.memo()
@@ -39,7 +39,7 @@
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
     # XXX pick better length, maybe
-    result, consumed = runicode.str_decode_utf8_escape(
+    result, consumed = rutf8.str_decode_utf8_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         unicodedata_handler=unicodedata_handler)
@@ -47,7 +47,7 @@
 
 def decode_raw_unicode_escape(space, string):
     # XXX pick better length, maybe
-    result, consumed = runicode.str_decode_raw_utf8_escape(
+    result, consumed = rutf8.str_decode_raw_utf8_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space))
     return result
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -5,7 +5,7 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from rpython.rlib.runicode import UNICHR, unichr_as_utf8
+from rpython.rlib.rutf8 import unichr_as_utf8
 from rpython.rlib.rfloat import isnan, isinf, round_double
 from rpython.rlib import rfloat
 import __builtin__
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -7,8 +7,8 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
-    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii,
-    check_ascii, AsciiCheckError)
+    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
+from rpython.rlib import rutf8
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
@@ -579,8 +579,8 @@
         return unicode_from_encoded_object(space, w_bytes, encoding, "strict")
     s = space.bytes_w(w_bytes)
     try:
-        check_ascii(s)
-    except AsciiCheckError:
+        rutf8.check_ascii(s)
+    except rutf8.AsciiCheckError:
         # raising UnicodeDecodeError is messy, "please crash for me"
         return utf8_from_encoded_object(space, w_bytes, "ascii", "strict")
     return W_UnicodeObject(s)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -42,36 +42,6 @@
         if len(u) == 1:
             return ord(u[0])
         raise TypeError
-
-def unichr_as_utf8(code):
-    """ Encode code (numeric value) as utf8 encoded string
-    """
-    if code < 0:
-        raise ValueError
-    if code < 0x80:
-        # Encode ASCII
-        return chr(code)
-    if code < 0x0800:
-        # Encode Latin-1
-        return chr((0xc0 | (code >> 6))) + chr((0x80 | (code & 0x3f)))
-    if code < 0x10000:
-        return (chr((0xe0 | (code >> 12))) +
-                chr((0x80 | ((code >> 6) & 0x3f))) +
-                chr((0x80 | (code & 0x3f))))
-    if code < 0x10ffff:
-        return (chr((0xf0 | (code >> 18))) +
-                chr((0x80 | ((code >> 12) & 0x3f))) +
-                chr((0x80 | ((code >> 6) & 0x3f))) +
-                chr((0x80 | (code & 0x3f))))
-    raise ValueError
-
-class AsciiCheckError(Exception):
-    pass
-
-def check_ascii(s):
-    for i in range(0, len(s)):
-        if ord(s[i]) & 0x80:
-            raise AsciiCheckError
         
 if MAXUNICODE > sys.maxunicode:
     # A version of unichr which allows codes outside the BMP
@@ -1407,129 +1377,6 @@
 
     return builder.build(), pos
 
-def str_decode_utf8_escape(s, size, errors, final=False,
-                              errorhandler=None,
-                              unicodedata_handler=None):
-    if errorhandler is None:
-        errorhandler = default_unicode_error_decode
-
-    if size == 0:
-        return '', 0
-
-    builder = StringBuilder(size)
-    pos = 0
-    while pos < size:
-        ch = s[pos]
-
-        # Non-escape characters are interpreted as Unicode ordinals
-        if ch != '\\':
-            builder.append(ch)
-            pos += 1
-            continue
-
-        # - Escapes
-        pos += 1
-        if pos >= size:
-            message = "\\ at end of string"
-            res, pos = errorhandler(errors, "unicodeescape",
-                                    message, s, pos-1, size)
-            builder.append(res)
-            continue
-
-        ch = s[pos]
-        pos += 1
-        # \x escapes
-        if ch == '\n': pass
-        elif ch == '\\': builder.append('\\')
-        elif ch == '\'': builder.append('\'')
-        elif ch == '\"': builder.append('\"')
-        elif ch == 'b' : builder.append('\b')
-        elif ch == 'f' : builder.append('\f')
-        elif ch == 't' : builder.append('\t')
-        elif ch == 'n' : builder.append('\n')
-        elif ch == 'r' : builder.append('\r')
-        elif ch == 'v' : builder.append('\v')
-        elif ch == 'a' : builder.append('\a')
-        elif '0' <= ch <= '7':
-            xxx
-            x = ord(ch) - ord('0')
-            if pos < size:
-                ch = s[pos]
-                if '0' <= ch <= '7':
-                    pos += 1
-                    x = (x<<3) + ord(ch) - ord('0')
-                    if pos < size:
-                        ch = s[pos]
-                        if '0' <= ch <= '7':
-                            pos += 1
-                            x = (x<<3) + ord(ch) - ord('0')
-            builder.append(unichr(x))
-        # hex escapes
-        # \xXX
-        elif ch == 'x':
-            xxx
-            digits = 2
-            message = "truncated \\xXX escape"
-            pos = hexescape(builder, s, pos, digits,
-                            "unicodeescape", errorhandler, message, errors)
-
-        # \uXXXX
-        elif ch == 'u':
-            xxx
-            digits = 4
-            message = "truncated \\uXXXX escape"
-            pos = hexescape(builder, s, pos, digits,
-                            "unicodeescape", errorhandler, message, errors)
-
-        #  \UXXXXXXXX
-        elif ch == 'U':
-            xxx
-            digits = 8
-            message = "truncated \\UXXXXXXXX escape"
-            pos = hexescape(builder, s, pos, digits,
-                            "unicodeescape", errorhandler, message, errors)
-
-        # \N{name}
-        elif ch == 'N' and unicodedata_handler is not None:
-            xxx
-            message = "malformed \\N character escape"
-            look = pos
-
-            if look < size and s[look] == '{':
-                # look for the closing brace
-                while look < size and s[look] != '}':
-                    look += 1
-                if look < size and s[look] == '}':
-                    # found a name.  look it up in the unicode database
-                    message = "unknown Unicode character name"
-                    name = s[pos+1:look]
-                    code = unicodedata_handler.call(name)
-                    if code < 0:
-                        res, pos = errorhandler(errors, "unicodeescape",
-                                                message, s, pos-1, look+1)
-                        builder.append(res)
-                        continue
-                    pos = look + 1
-                    if code <= MAXUNICODE:
-                        builder.append(UNICHR(code))
-                    else:
-                        code -= 0x10000L
-                        builder.append(unichr(0xD800 + (code >> 10)))
-                        builder.append(unichr(0xDC00 + (code & 0x03FF)))
-                else:
-                    res, pos = errorhandler(errors, "unicodeescape",
-                                            message, s, pos-1, look+1)
-                    builder.append(res)
-            else:
-                res, pos = errorhandler(errors, "unicodeescape",
-                                        message, s, pos-1, look+1)
-                builder.append(res)
-        else:
-            builder.append('\\')
-            builder.append(ch)
-
-    return builder.build(), pos
-
 def make_unicode_escape_function(pass_printable=False, unicode_output=False,
                                  quotes=False, prefix=None):
     # Python3 has two similar escape functions: One to implement
@@ -1650,54 +1497,6 @@
 # ____________________________________________________________
 # Raw unicode escape
 
-def str_decode_raw_utf8_escape(s, size, errors, final=False,
-                               errorhandler=None):
-    if errorhandler is None:
-        errorhandler = default_unicode_error_decode
-    if size == 0:
-        return '', 0
-    result = StringBuilder(size)
-    pos = 0
-    while pos < size:
-        ch = s[pos]
-
-        # Non-escape characters are interpreted as Unicode ordinals
-        if ch != '\\':
-            result.append(ch)
-            pos += 1
-            continue
-
-        # \u-escapes are only interpreted iff the number of leading
-        # backslashes is odd
-        bs = pos
-        while pos < size:
-            pos += 1
-            if pos == size or s[pos] != '\\':
-                break
-            result.append('\\')
-
-        # we have a backslash at the end of the string, stop here
-        if pos >= size:
-            result.append('\\')
-            break
-
-        if ((pos - bs) & 1 == 0 or
-            pos >= size or
-            (s[pos] != 'u' and s[pos] != 'U')):
-            result.append('\\')
-            result.append(s[pos])
-            pos += 1
-            continue
-
-        digits = 4 if s[pos] == 'u' else 8
-        message = "truncated \\uXXXX"
-        pos += 1
-        xxx # change hexescape to deal with utf8
-        pos = hexescape(result, s, pos, digits,
-                        "rawunicodeescape", errorhandler, message, errors)
-
-    return result.build(), pos
-
 def str_decode_raw_unicode_escape(s, size, errors, final=False,
                                   errorhandler=None):
     if errorhandler is None:
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rutf8.py
@@ -0,0 +1,203 @@
+
+from rpython.rlib.rstring import StringBuilder
+
+def unichr_as_utf8(code):
+    """ Encode code (numeric value) as utf8 encoded string
+    """
+    if code < 0:
+        raise ValueError
+    if code < 0x80:
+        # Encode ASCII
+        return chr(code)
+    if code < 0x0800:
+        # Encode Latin-1
+        return chr((0xc0 | (code >> 6))) + chr((0x80 | (code & 0x3f)))
+    if code < 0x10000:
+        return (chr((0xe0 | (code >> 12))) +
+                chr((0x80 | ((code >> 6) & 0x3f))) +
+                chr((0x80 | (code & 0x3f))))
+    if code < 0x10ffff:
+        return (chr((0xf0 | (code >> 18))) +
+                chr((0x80 | ((code >> 12) & 0x3f))) +
+                chr((0x80 | ((code >> 6) & 0x3f))) +
+                chr((0x80 | (code & 0x3f))))
+    raise ValueError
+
+class AsciiCheckError(Exception):
+    pass
+
+def check_ascii(s):
+    for i in range(0, len(s)):
+        if ord(s[i]) & 0x80:
+            raise AsciiCheckError
+
+def str_decode_raw_utf8_escape(s, size, errors, final=False,
+                               errorhandler=None):
+    if errorhandler is None:
+        errorhandler = None # default_unicode_error_decode
+    if size == 0:
+        return '', 0
+    result = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            result.append(ch)
+            pos += 1
+            continue
+
+        # \u-escapes are only interpreted iff the number of leading
+        # backslashes is odd
+        bs = pos
+        while pos < size:
+            pos += 1
+            if pos == size or s[pos] != '\\':
+                break
+            result.append('\\')
+
+        # we have a backslash at the end of the string, stop here
+        if pos >= size:
+            result.append('\\')
+            break
+
+        if ((pos - bs) & 1 == 0 or
+            pos >= size or
+            (s[pos] != 'u' and s[pos] != 'U')):
+            result.append('\\')
+            result.append(s[pos])
+            pos += 1
+            continue
+
+        digits = 4 if s[pos] == 'u' else 8
+        message = "truncated \\uXXXX"
+        pos += 1
+        xxx # change hexescape to deal with utf8
+        pos = hexescape(result, s, pos, digits,
+                        "rawunicodeescape", errorhandler, message, errors)
+
+    return result.build(), pos
+
+def str_decode_utf8_escape(s, size, errors, final=False,
+                              errorhandler=None,
+                              unicodedata_handler=None):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+
+    if size == 0:
+        return '', 0
+
+    builder = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = s[pos]
+
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            builder.append(ch)
+            pos += 1
+            continue
+
+        # - Escapes
+        pos += 1
+        if pos >= size:
+            message = "\\ at end of string"
+            res, pos = errorhandler(errors, "unicodeescape",
+                                    message, s, pos-1, size)
+            builder.append(res)
+            continue
+
+        ch = s[pos]
+        pos += 1
+        # \x escapes
+        if ch == '\n': pass
+        elif ch == '\\': builder.append('\\')
+        elif ch == '\'': builder.append('\'')
+        elif ch == '\"': builder.append('\"')
+        elif ch == 'b' : builder.append('\b')
+        elif ch == 'f' : builder.append('\f')
+        elif ch == 't' : builder.append('\t')
+        elif ch == 'n' : builder.append('\n')
+        elif ch == 'r' : builder.append('\r')
+        elif ch == 'v' : builder.append('\v')
+        elif ch == 'a' : builder.append('\a')
+        elif '0' <= ch <= '7':
+            xxx
+            x = ord(ch) - ord('0')
+            if pos < size:
+                ch = s[pos]
+                if '0' <= ch <= '7':
+                    pos += 1
+                    x = (x<<3) + ord(ch) - ord('0')
+                    if pos < size:
+                        ch = s[pos]
+                        if '0' <= ch <= '7':
+                            pos += 1
+                            x = (x<<3) + ord(ch) - ord('0')
+            builder.append(unichr(x))
+        # hex escapes
+        # \xXX
+        elif ch == 'x':
+            xxx
+            digits = 2
+            message = "truncated \\xXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \uXXXX
+        elif ch == 'u':
+            xxx
+            digits = 4
+            message = "truncated \\uXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        #  \UXXXXXXXX
+        elif ch == 'U':
+            xxx
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \N{name}
+        elif ch == 'N' and unicodedata_handler is not None:
+            xxx
+            message = "malformed \\N character escape"
+            look = pos
+
+            if look < size and s[look] == '{':
+                # look for the closing brace
+                while look < size and s[look] != '}':
+                    look += 1
+                if look < size and s[look] == '}':
+                    # found a name.  look it up in the unicode database
+                    message = "unknown Unicode character name"
+                    name = s[pos+1:look]
+                    code = unicodedata_handler.call(name)
+                    if code < 0:
+                        res, pos = errorhandler(errors, "unicodeescape",
+                                                message, s, pos-1, look+1)
+                        builder.append(res)
+                        continue
+                    pos = look + 1
+                    if code <= MAXUNICODE:
+                        builder.append(UNICHR(code))
+                    else:
+                        code -= 0x10000L
+                        builder.append(unichr(0xD800 + (code >> 10)))
+                        builder.append(unichr(0xDC00 + (code & 0x03FF)))
+                else:
+                    res, pos = errorhandler(errors, "unicodeescape",
+                                            message, s, pos-1, look+1)
+                    builder.append(res)
+            else:
+                res, pos = errorhandler(errors, "unicodeescape",
+                                        message, s, pos-1, look+1)
+                builder.append(res)
+        else:
+            builder.append('\\')
+            builder.append(ch)
+
+    return builder.build(), pos

From pypy.commits at gmail.com  Tue Feb 21 07:20:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 04:20:13 -0800 (PST)
Subject: [pypy-commit] pypy default: Seeing again the crash
 "ExecutionContext instances should not be seen
Message-ID: <58ac307d.11162e0a.87141.8337@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90257:bd3fb63c58c8
Date: 2017-02-21 13:19 +0100
http://bitbucket.org/pypy/pypy/changeset/bd3fb63c58c8/

Log:	Seeing again the crash "ExecutionContext instances should not be
	seen during translation". At least in this case I could track it to
	this. Unsure why it was set in the first place, but fix it the easy
	way.

diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -33,6 +33,11 @@
     def getallvalues(self):
         return {0: self._value}
 
+    def _cleanup_(self):
+        # should still be unfilled at this point during translation.
+        # but in some corner cases it is not...  unsure why
+        self._value = None
+
 
 def make_weak_value_dictionary(space, keytype, valuetype):
     "NOT_RPYTHON"

From pypy.commits at gmail.com  Tue Feb 21 07:20:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 04:20:11 -0800 (PST)
Subject: [pypy-commit] pypy default: Call fsdecode_w() also in link() and
 readlink(), otherwise we can't pass
Message-ID: <58ac307b.8fdb190a.4c3c1.86eb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90256:58d7aea67974
Date: 2017-02-21 12:34 +0100
http://bitbucket.org/pypy/pypy/changeset/58d7aea67974/

Log:	Call fsdecode_w() also in link() and readlink(), otherwise we can't
	pass a unicode containing a non-ascii char

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -698,9 +698,10 @@
     import signal
     rposix.kill(os.getpid(), signal.SIGABRT)
 
- at unwrap_spec(src='str0', dst='str0')
-def link(space, src, dst):
+def link(space, w_src, w_dst):
     "Create a hard link to a file."
+    src = fsencode_w(space, w_src)
+    dst = fsencode_w(space, w_dst)
     try:
         os.link(src, dst)
     except OSError as e:
@@ -713,9 +714,9 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(path='str0')
-def readlink(space, path):
+def readlink(space, w_path):
     "Return a string representing the path to which the symbolic link points."
+    path = fsencode_w(space, w_path)
     try:
         result = os.readlink(path)
     except OSError as e:

From pypy.commits at gmail.com  Tue Feb 21 07:31:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 04:31:57 -0800 (PST)
Subject: [pypy-commit] pypy default: Use fsencode_w() instead of str0 at a
 few more places (this is really a bug fix)
Message-ID: <58ac333d.8c2b190a.255d7.9023@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90258:90917511788e
Date: 2017-02-21 13:31 +0100
http://bitbucket.org/pypy/pypy/changeset/90917511788e/

Log:	Use fsencode_w() instead of str0 at a few more places (this is
	really a bug fix)

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -857,18 +857,20 @@
         except OSError as e:
             raise wrap_oserror(space, e)
 
- at unwrap_spec(mode=int, path='str0')
-def spawnv(space, mode, path, w_args):
-    args = [space.str0_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+ at unwrap_spec(mode=int)
+def spawnv(space, mode, w_path, w_args):
+    path = fsencode_w(space, w_path)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     try:
         ret = os.spawnv(mode, path, args)
     except OSError as e:
         raise wrap_oserror(space, e)
     return space.newint(ret)
 
- at unwrap_spec(mode=int, path='str0')
-def spawnve(space, mode, path, w_args, w_env):
-    args = [space.str0_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+ at unwrap_spec(mode=int)
+def spawnve(space, mode, w_path, w_args, w_env):
+    path = fsencode_w(space, w_path)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     env = _env2interp(space, w_env)
     try:
         ret = os.spawnve(mode, path, args, env)
@@ -967,12 +969,12 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(path='str0')
-def chroot(space, path):
+def chroot(space, w_path):
     """ chroot(path)
 
     Change root directory to path.
     """
+    path = fsencode_w(space, w_path)
     try:
         os.chroot(path)
     except OSError as e:
@@ -1257,8 +1259,8 @@
         raise wrap_oserror(space, e)
     return space.newint(res)
 
- at unwrap_spec(path='str0')
-def pathconf(space, path, w_name):
+def pathconf(space, w_path, w_name):
+    path = fsencode_w(space, w_path)
     num = confname_w(space, w_name, os.pathconf_names)
     try:
         res = os.pathconf(path, num)
@@ -1274,18 +1276,20 @@
         raise wrap_oserror(space, e)
     return space.newtext(res)
 
- at unwrap_spec(path='str0', uid=c_uid_t, gid=c_gid_t)
-def chown(space, path, uid, gid):
+ at unwrap_spec(uid=c_uid_t, gid=c_gid_t)
+def chown(space, w_path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid."""
+    path = fsencode_w(space, w_path)
     try:
         os.chown(path, uid, gid)
     except OSError as e:
         raise wrap_oserror(space, e, path)
 
- at unwrap_spec(path='str0', uid=c_uid_t, gid=c_gid_t)
-def lchown(space, path, uid, gid):
+ at unwrap_spec(uid=c_uid_t, gid=c_gid_t)
+def lchown(space, w_path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid.
 This function will not follow symbolic links."""
+    path = fsencode_w(space, w_path)
     try:
         os.lchown(path, uid, gid)
     except OSError as e:

From pypy.commits at gmail.com  Tue Feb 21 07:50:20 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 21 Feb 2017 04:50:20 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-2: fix two bugs in
 3170ca56ec4a:
Message-ID: <58ac378c.12ce190a.9bd3e.921f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-2
Changeset: r90260:fec88c2114cb
Date: 2017-02-19 22:02 +0100
http://bitbucket.org/pypy/pypy/changeset/fec88c2114cb/

Log:	fix two bugs in 3170ca56ec4a:

	- wraparound problems in integer_fits
	- pending fields were broken by the change from short to regular
	integers

diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1004,8 +1004,8 @@
         if pendingfields:
             for i in range(len(pendingfields)):
                 lldescr = pendingfields[i].lldescr
-                num = pendingfields[i].num
-                fieldnum = pendingfields[i].fieldnum
+                num = rarithmetic.widen(pendingfields[i].num)
+                fieldnum = rarithmetic.widen(pendingfields[i].fieldnum)
                 itemindex = pendingfields[i].itemindex
                 descr = annlowlevel.cast_base_ptr_to_instance(AbstractDescr,
                                                               lldescr)
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -22,15 +22,18 @@
 """
 
 from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib import objectmodel
+from rpython.rlib import objectmodel, rarithmetic
 
 class TagOverflow(Exception):
     pass
 
 def integer_fits(item):
-    item *= 2
+    try:
+        item = rarithmetic.ovfcheck(item * 2)
+    except OverflowError:
+        return False
     if item < 0:
-       item = ~item
+        item = ~item # can't overflow
     # we can fit up to 22 content bits into 1-3 bytes (24 bits, with 2 continuation
     # bits)
     return item < 2 ** 22
diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -1,3 +1,4 @@
+import sys
 import pytest
 from rpython.jit.metainterp.resumecode import create_numbering,\
     unpack_numbering, Reader, Writer, TagOverflow
@@ -7,12 +8,10 @@
 
 def test_tag_overflow():
     w = Writer()
-    with pytest.raises(TagOverflow):
-        w.append_int(2**21)
-    with pytest.raises(TagOverflow):
-        w.append_int(-2**22)
-    with pytest.raises(TagOverflow):
-        w.append_int(-2**21-1)
+    for bad in [2 ** 21, -2**22, -2**21-1,
+                sys.maxint, -sys.maxint-1]:
+        with pytest.raises(TagOverflow):
+            w.append_int(bad)
 
 examples = [
     [1, 2, 3, 4, 257, 10000, 13, 15],

From pypy.commits at gmail.com  Tue Feb 21 07:50:17 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 21 Feb 2017 04:50:17 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-2: merge 3170ca56ec4a: make
 use of the full three bytes that resumecode uses by allowing more values
Message-ID: <58ac3789.489a190a.2c962.8c85@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-2
Changeset: r90259:5209cb965cae
Date: 2017-02-21 13:45 +0100
http://bitbucket.org/pypy/pypy/changeset/5209cb965cae/

Log:	merge 3170ca56ec4a: make use of the full three bytes that resumecode
	uses by allowing more values

	(needs some fixes, see next commit)

diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
--- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -81,9 +81,9 @@
             if index == -1:
                 # XXX XXX XXX fix length!
                 continue # just skip it, if the descr is not encodable
-            numb_state.append_short(tag_box(box1, liveboxes_from_env, memo))
+            numb_state.append_int(tag_box(box1, liveboxes_from_env, memo))
             numb_state.append_int(index)
-            numb_state.append_short(tag_box(box2, liveboxes_from_env, memo))
+            numb_state.append_int(tag_box(box2, liveboxes_from_env, memo))
     else:
         numb_state.append_int(0)
 
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -13,6 +13,7 @@
 from rpython.rtyper.rclass import OBJECTPTR
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp import resumecode
+TagOverflow = resumecode.TagOverflow
 
 
 # Logic to encode the chain of frames and the state of the boxes at a
@@ -124,18 +125,14 @@
 
 TAGMASK = 3
 
-class TagOverflow(Exception):
-    pass
-
 def tag(value, tagbits):
     assert 0 <= tagbits <= 3
-    sx = value >> 13
-    if sx != 0 and sx != -1:
-        raise TagOverflow
-    return rffi.r_short(value<<2|tagbits)
+    result = value<<2|tagbits
+    if not resumecode.integer_fits(result):
+        raise resumecode.TagOverflow
+    return result
 
 def untag(value):
-    value = rarithmetic.widen(value)
     tagbits = value & TAGMASK
     return value >> 2, tagbits
 
@@ -249,7 +246,7 @@
                     tagged = tag(num_boxes, TAGBOX)
                     num_boxes += 1
                 liveboxes[box] = tagged
-            numb_state.append_short(tagged)
+            numb_state.append_int(tagged)
         numb_state.num_boxes = num_boxes
         numb_state.num_virtuals = num_virtuals
 
@@ -559,8 +556,8 @@
                 fieldbox = optimizer.get_box_replacement(fieldbox)
                 #descr, box, fieldbox, itemindex = pending_setfields[i]
                 lldescr = annlowlevel.cast_instance_to_base_ptr(descr)
-                num = self._gettagged(box)
-                fieldnum = self._gettagged(fieldbox)
+                num = rffi.r_short(self._gettagged(box))
+                fieldnum = rffi.r_short(self._gettagged(fieldbox))
                 # the index is limited to 2147483647 (64-bit machines only)
                 if itemindex > 2147483647:
                     raise TagOverflow
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -1,4 +1,3 @@
-
 """ Resume bytecode. It goes as following:
 
   # ----- resume section
@@ -19,11 +18,24 @@
 
   # ----- optimization section
   <more code>                                      further sections according to bridgeopt.py
+
 """
 
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import objectmodel
 
+class TagOverflow(Exception):
+    pass
+
+def integer_fits(item):
+    item *= 2
+    if item < 0:
+       item = ~item
+    # we can fit up to 22 content bits into 1-3 bytes (24 bits, with 2 continuation
+    # bits)
+    return item < 2 ** 22
+
+
 NUMBERINGP = lltype.Ptr(lltype.GcForwardReference())
 NUMBERING = lltype.GcStruct('Numbering',
                             ('code', lltype.Array(rffi.UCHAR)))
@@ -31,10 +43,13 @@
 NULL_NUMBER = lltype.nullptr(NUMBERING)
 
 def append_numbering(lst, item):
-    item = rffi.cast(lltype.Signed, item)
+    # item is just an int
+    assert isinstance(item, int)
+    if not integer_fits(item):
+        raise TagOverflow
     item *= 2
     if item < 0:
-        item = -1 - item
+       item = ~item
 
     assert item >= 0
     if item < 2**7:
@@ -43,7 +58,7 @@
         lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
         lst.append(rffi.cast(rffi.UCHAR, item >> 7))
     else:
-        assert item < 2**16
+        assert item < 2**22
         lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
         lst.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
         lst.append(rffi.cast(rffi.UCHAR, item >> 14))
@@ -85,15 +100,10 @@
         self.grow(size)
         self.items = 0
 
-    def append_short(self, item):
+    def append_int(self, item):
         self.items += 1
         append_numbering(self.current, item)
 
-    def append_int(self, item):
-        short = rffi.cast(rffi.SHORT, item)
-        assert rffi.cast(lltype.Signed, short) == item
-        return self.append_short(short)
-
     def create_numbering(self):
         numb = lltype.malloc(NUMBERING, len(self.current))
         for i, elt in enumerate(self.current):
@@ -105,7 +115,7 @@
 
     def patch_current_size(self, index):
         # mess :-(
-        assert rffi.cast(lltype.Signed, self.current[index]) == 0
+        assert self.current[index] == 0
         l = []
         append_numbering(l, self.items)
         self.current = l + self.current[1:]
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -111,10 +111,10 @@
 
 
 def test_tag():
-    assert tag(3, 1) == rffi.r_short(3<<2|1)
-    assert tag(-3, 2) == rffi.r_short(-3<<2|2)
-    assert tag((1<<13)-1, 3) == rffi.r_short(((1<<15)-1)|3)
-    assert tag(-1<<13, 3) == rffi.r_short((-1<<15)|3)
+    assert tag(3, 1) == 3<<2|1
+    assert tag(-3, 2) == -3<<2|2
+    assert tag((1<<13)-1, 3) == ((1<<15)-1)|3
+    assert tag(-1<<13, 3) ==(-1<<15)|3
     py.test.raises(AssertionError, tag, 3, 5)
     py.test.raises(TagOverflow, tag, 1<<13, 0)
     py.test.raises(TagOverflow, tag, (1<<13)+1, 0)
diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -1,9 +1,19 @@
+import pytest
 from rpython.jit.metainterp.resumecode import create_numbering,\
-    unpack_numbering, Reader, Writer
+    unpack_numbering, Reader, Writer, TagOverflow
 from rpython.rtyper.lltypesystem import lltype
 
 from hypothesis import strategies, given, example
 
+def test_tag_overflow():
+    w = Writer()
+    with pytest.raises(TagOverflow):
+        w.append_int(2**21)
+    with pytest.raises(TagOverflow):
+        w.append_int(-2**22)
+    with pytest.raises(TagOverflow):
+        w.append_int(-2**21-1)
+
 examples = [
     [1, 2, 3, 4, 257, 10000, 13, 15],
     [1, 2, 3, 4],
@@ -12,7 +22,7 @@
 ]
 
 def hypothesis_and_examples(func):
-    func = given(strategies.lists(strategies.integers(-2**15, 2**15-1)))(func)
+    func = given(strategies.lists(strategies.integers(-2**21, 2**21-1)))(func)
     for ex in examples:
         func = example(ex)(func)
     return func
@@ -60,3 +70,5 @@
         n = w.create_numbering()
         assert unpack_numbering(n)[1:] == l
         assert unpack_numbering(n)[0] == middle + 1
+
+

From pypy.commits at gmail.com  Tue Feb 21 07:51:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 04:51:04 -0800 (PST)
Subject: [pypy-commit] pypy default: fix for translations without threads
Message-ID: <58ac37b8.8bcf190a.7f7fc.8b91@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90261:b22622b0bc15
Date: 2017-02-21 13:34 +0100
http://bitbucket.org/pypy/pypy/changeset/b22622b0bc15/

Log:	fix for translations without threads

diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -282,6 +282,8 @@
     else:
         assert ec.cpyext_gilstate_counter_noleave == 0
         assert oldstate == PyGILState_UNLOCKED
+        assert space.config.translation.thread
+        #      ^^^ otherwise, we should not reach this case
         ec.cpyext_threadstate_is_current = False
         space.threadlocals.leave_thread(space)
 

From pypy.commits at gmail.com  Tue Feb 21 07:57:04 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 21 Feb 2017 04:57:04 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-2: merge default
Message-ID: <58ac3920.514d2e0a.6102d.9286@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-2
Changeset: r90262:1764a122b2bd
Date: 2017-02-21 13:56 +0100
http://bitbucket.org/pypy/pypy/changeset/1764a122b2bd/

Log:	merge default

diff too long, truncating to 2000 out of 28018 lines

diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,9 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break        
+                break
+        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+                                                 'include')
 
     if args:
         vals = []
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -604,7 +604,8 @@
         """
         # hack for performance: if restype is a "simple" primitive type, don't
         # allocate the buffer because it's going to be thrown away immediately
-        if self._is_primitive(restype) and not restype._is_pointer_like():
+        if (self._is_primitive(restype) and restype._type_ != '?'
+            and not restype._is_pointer_like()):
             return result
         #
         shape = restype._ffishape_
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -112,11 +112,20 @@
     def _make_cmp(name):
         cmpfunc = getattr(operator, name)
         def cmp(self, other):
-            if isinstance(other, CTypesData):
+            v_is_ptr = not isinstance(self, CTypesGenericPrimitive)
+            w_is_ptr = (isinstance(other, CTypesData) and
+                           not isinstance(other, CTypesGenericPrimitive))
+            if v_is_ptr and w_is_ptr:
                 return cmpfunc(self._convert_to_address(None),
                                other._convert_to_address(None))
+            elif v_is_ptr or w_is_ptr:
+                return NotImplemented
             else:
-                return NotImplemented
+                if isinstance(self, CTypesGenericPrimitive):
+                    self = self._value
+                if isinstance(other, CTypesGenericPrimitive):
+                    other = other._value
+                return cmpfunc(self, other)
         cmp.func_name = name
         return cmp
 
@@ -128,7 +137,7 @@
     __ge__ = _make_cmp('__ge__')
 
     def __hash__(self):
-        return hash(type(self)) ^ hash(self._convert_to_address(None))
+        return hash(self._convert_to_address(None))
 
     def _to_string(self, maxlen):
         raise TypeError("string(): %r" % (self,))
@@ -137,14 +146,8 @@
 class CTypesGenericPrimitive(CTypesData):
     __slots__ = []
 
-    def __eq__(self, other):
-        return self is other
-
-    def __ne__(self, other):
-        return self is not other
-
     def __hash__(self):
-        return object.__hash__(self)
+        return hash(self._value)
 
     def _get_own_repr(self):
         return repr(self._from_ctypes(self._value))
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -188,6 +188,7 @@
 
 .. py:function:: wrap(x)
 
+   **Deprecated! Eventually this method should disappear.**
    Returns a wrapped object that is a reference to the interpreter-level object
    :py:obj:`x`. This can be used either on simple immutable objects (integers,
    strings, etc) to create a new wrapped object, or on instances of :py:class:`W_Root`
@@ -196,6 +197,35 @@
    be directly exposed to application-level code in this way - functions, frames,
    code objects, etc.
 
+.. py:function:: newint(i)
+
+   Creates a wrapped object holding an integral value. `newint` creates an object
+   of type `W_IntObject`.
+
+.. py:function:: newlong(l)
+
+   Creates a wrapped object holding an integral value. The main difference to newint
+   is the type of the argument (which is rpython.rlib.rbigint.rbigint). On PyPy3 this
+   method will return an :py:class:`int` (PyPy2 it returns a :py:class:`long`).
+
+.. py:function:: newbytes(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`bytes` (both on PyPy2 and PyPy3).
+
+.. py:function:: newtext(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`str`.  On PyPy3 this will return a wrapped unicode
+   object. The object will hold a utf-8-nosg decoded value of `t`.
+   The "utf-8-nosg" codec used here is slightly different from the
+   "utf-8" implemented in Python 2 or Python 3: it is defined as utf-8
+   without any special handling of surrogate characters.  They are
+   encoded using the same three-bytes sequence that encodes any char in
+   the range from ``'\u0800'`` to ``'\uffff'``.
+
+   PyPy2 will return a bytestring object. No encoding/decoding steps will be applied.
+
 .. py:function:: newbool(b)
 
    Creates a wrapped :py:class:`bool` object from an :ref:`interpreter-level <interpreter-level>`
@@ -217,15 +247,18 @@
 
    Creates a new slice object.
 
-.. py:function:: newstring(asciilist)
+.. py:function:: newunicode(ustr)
 
-   Creates a string from a list of wrapped integers. Note that this may not be
-   a very useful method; usually you can just write ``space.wrap("mystring")``.
+   Creates a Unicode string from an rpython unicode string.
+   This method may disappear soon and be replaced by :py:function:`newutf8()`.
 
-.. py:function:: newunicode(codelist)
+.. py:function:: newutf8(bytestr)
 
-   Creates a Unicode string from a list of integers (code points).
+   Creates a Unicode string from an rpython byte string, decoded as
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
 
+Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
+`pypy/objspace/std/objspace.py`.
 
 Conversions from Application Level to Interpreter Level
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -258,11 +291,28 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: text_w(w_x)
+
+   Takes an application level :py:class:`str` and converts it to a rpython byte string.
+   PyPy3 this method will return an utf-8-nosg encoded result.
+
+.. py:function:: bytes_w(w_x)
+
+   Takes an application level :py:class:`bytes` (PyPy2 this equals `str`) and returns a rpython
+   byte string.
+
 .. py:function:: str_w(w_x)
 
+   **Deprecated. use text_w or bytes_w instead**
    If :py:obj:`w_x` is an application-level string, return an interpreter-level string.
    Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: unicode_w(w_x)
+
+   Takes an application level :py:class:`unicode` and return an
+   interpreter-level unicode string.  This method may disappear soon and
+   be replaced by :py:function:`text_w()`.
+
 .. py:function:: float_w(w_x)
 
    If :py:obj:`w_x` is an application-level float, integer or long, return an
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -142,3 +142,15 @@
 it is the standard ``OrderedDict.move_to_end()`` method, but the
 behavior is also available on Python 2.x or for the ``dict`` type by
 calling ``__pypy__.move_to_end(dict, key, last=True)``.
+
+.. branch: space-newtext
+
+Internal refactoring of ``space.wrap()``, which is now replaced with
+explicitly-typed methods.  Notably, there are now ``space.newbytes()``
+and ``space.newtext()``: these two methods are identical on PyPy 2.7 but
+not on PyPy 3.x.  The latter is used to get an app-level unicode string
+by decoding the RPython string, assumed to be utf-8.
+
+.. branch: fix_bool_restype
+
+Fix for ``ctypes.c_bool``-returning ctypes functions
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -31,9 +31,9 @@
 
 def create_entry_point(space, w_dict):
     if w_dict is not None: # for tests
-        w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
-        w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
-        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
+        w_entry_point = space.getitem(w_dict, space.newtext('entry_point'))
+        w_run_toplevel = space.getitem(w_dict, space.newtext('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.newtext('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
         hashfunc = space.config.objspace.hash
     else:
@@ -65,8 +65,8 @@
         try:
             try:
                 space.startup()
-                w_executable = space.wrap(argv[0])
-                w_argv = space.newlist([space.wrap(s) for s in argv[1:]])
+                w_executable = space.newtext(argv[0])
+                w_argv = space.newlist([space.newtext(s) for s in argv[1:]])
                 w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
                 exitcode = space.int_w(w_exitcode)
                 # try to pull it all in
@@ -76,7 +76,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
             try:
@@ -84,7 +84,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
 
@@ -123,7 +123,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap(home), w_initstdio],
+            space.appexec([w_path, space.newtext(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path
@@ -141,7 +141,7 @@
             if verbose:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
             if must_leave:
@@ -180,11 +180,11 @@
     def _pypy_execute_source(source, c_argument):
         try:
             w_globals = space.newdict(module=True)
-            space.setitem(w_globals, space.wrap('__builtins__'),
+            space.setitem(w_globals, space.newtext('__builtins__'),
                           space.builtin_modules['__builtin__'])
-            space.setitem(w_globals, space.wrap('c_argument'),
-                          space.wrap(c_argument))
-            space.appexec([space.wrap(source), w_globals], """(src, glob):
+            space.setitem(w_globals, space.newtext('c_argument'),
+                          space.newint(c_argument))
+            space.appexec([space.newtext(source), w_globals], """(src, glob):
                 import sys
                 stmt = compile(src, 'c callback', 'exec')
                 if not hasattr(sys, '_pypy_execute_source'):
@@ -195,7 +195,7 @@
         except OperationError as e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space))
-            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
 
@@ -323,7 +323,7 @@
         # obscure hack to stuff the translation options into the translated PyPy
         import pypy.module.sys
         options = make_dict(config)
-        wrapstr = 'space.wrap(%r)' % (options)
+        wrapstr = 'space.wrap(%r)' % (options) # import time
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -363,7 +363,7 @@
     i = 0
     for w_key in keys_w:
         try:
-            key = space.str_w(w_key)
+            key = space.text_w(w_key)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError, "keywords must be strings")
@@ -547,11 +547,11 @@
                         except IndexError:
                             name = '?'
                         else:
-                            w_enc = space.wrap(space.sys.defaultencoding)
-                            w_err = space.wrap("replace")
+                            w_enc = space.newtext(space.sys.defaultencoding)
+                            w_err = space.newtext("replace")
                             w_name = space.call_method(w_name, "encode", w_enc,
                                                        w_err)
-                            name = space.str_w(w_name)
+                            name = space.text_w(w_name)
                     break
         self.kwd_name = name
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -266,8 +266,8 @@
             else:
                 w_key = space.newtuple([obj, space.w_float])
         elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
+            w_real = space.getattr(obj, space.newtext("real"))
+            w_imag = space.getattr(obj, space.newtext("imag"))
             real = space.float_w(w_real)
             imag = space.float_w(w_imag)
             real_negzero = (real == 0.0 and
@@ -366,7 +366,7 @@
         space = self.space
         consts_w = [space.w_None] * space.len_w(w_consts)
         w_iter = space.iter(w_consts)
-        first = space.wrap(0)
+        first = space.newint(0)
         while True:
             try:
                 w_key = space.next(w_iter)
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -13,7 +13,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -48,10 +48,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
-
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+        assert fields == []
+
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -67,14 +67,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -93,12 +93,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -114,7 +114,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -143,16 +143,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)
+            space.newtext(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
@@ -196,7 +196,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -230,7 +230,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -258,7 +258,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expression)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -293,7 +293,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -390,26 +390,26 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FunctionDef)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -463,30 +463,30 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ClassDef)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.bases is None:
             bases_w = []
         else:
             bases_w = [node.to_object(space) for node in self.bases] # expr
         w_bases = space.newlist(bases_w)
-        space.setattr(w_node, space.wrap('bases'), w_bases)
+        space.setattr(w_node, space.newtext('bases'), w_bases)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -530,11 +530,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -572,11 +572,11 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -617,13 +617,13 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -663,15 +663,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AugAssign)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -719,19 +719,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Print)
         w_dest = self.dest.to_object(space) if self.dest is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('dest'), w_dest)
+        space.setattr(w_node, space.newtext('dest'), w_dest)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_nl = space.wrap(self.nl)  # bool
-        space.setattr(w_node, space.wrap('nl'), w_nl)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_nl = space.newbool(self.nl)  # bool
+        space.setattr(w_node, space.newtext('nl'), w_nl)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -780,25 +780,25 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_For)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -850,23 +850,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_While)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -914,23 +914,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_If)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -977,19 +977,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_With)
         w_context_expr = self.context_expr.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
+        space.setattr(w_node, space.newtext('context_expr'), w_context_expr)
         w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
+        space.setattr(w_node, space.newtext('optional_vars'), w_optional_vars)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1035,15 +1035,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
+        space.setattr(w_node, space.newtext('type'), w_type)
         w_inst = self.inst.to_object(space) if self.inst is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('inst'), w_inst)
+        space.setattr(w_node, space.newtext('inst'), w_inst)
         w_tback = self.tback.to_object(space) if self.tback is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('tback'), w_tback)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('tback'), w_tback)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1093,23 +1093,23 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.handlers is None:
             handlers_w = []
         else:
             handlers_w = [node.to_object(space) for node in self.handlers] # excepthandler
         w_handlers = space.newlist(handlers_w)
-        space.setattr(w_node, space.wrap('handlers'), w_handlers)
+        space.setattr(w_node, space.newtext('handlers'), w_handlers)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1158,17 +1158,17 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.finalbody is None:
             finalbody_w = []
         else:
             finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt
         w_finalbody = space.newlist(finalbody_w)
-        space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('finalbody'), w_finalbody)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1207,13 +1207,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Assert)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_msg = self.msg.to_object(space) if self.msg is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('msg'), w_msg)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('msg'), w_msg)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1255,11 +1255,11 @@
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1295,20 +1295,20 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = space.wrap(self.module)  # identifier
-        space.setattr(w_node, space.wrap('module'), w_module)
+        w_module = space.newtext_or_none(self.module)  # identifier
+        space.setattr(w_node, space.newtext('module'), w_module)
         if self.names is None:
             names_w = []
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_level = space.wrap(self.level)  # int
-        space.setattr(w_node, space.wrap('level'), w_level)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_level = space.newint(self.level)  # int
+        space.setattr(w_node, space.newtext('level'), w_level)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1351,15 +1351,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Exec)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_globals = self.globals.to_object(space) if self.globals is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('globals'), w_globals)
+        space.setattr(w_node, space.newtext('globals'), w_globals)
         w_locals = self.locals.to_object(space) if self.locals is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('locals'), w_locals)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('locals'), w_locals)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1398,13 +1398,13 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [space.wrap(node) for node in self.names] # identifier
+            names_w = [space.newtext(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1437,11 +1437,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1472,10 +1472,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Pass)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1502,10 +1502,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Break)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1532,10 +1532,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Continue)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1628,17 +1628,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BoolOp)
         w_op = boolop_to_class[self.op - 1]().to_object(space)  # boolop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1678,15 +1678,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BinOp)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_right = self.right.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('right'), w_right)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('right'), w_right)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1729,13 +1729,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_UnaryOp)
         w_op = unaryop_to_class[self.op - 1]().to_object(space)  # unaryop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_operand = self.operand.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('operand'), w_operand)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('operand'), w_operand)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1775,13 +1775,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Lambda)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1823,15 +1823,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_IfExp)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_orelse = self.orelse.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1883,17 +1883,17 @@
         else:
             keys_w = [node.to_object(space) for node in self.keys] # expr
         w_keys = space.newlist(keys_w)
-        space.setattr(w_node, space.wrap('keys'), w_keys)
+        space.setattr(w_node, space.newtext('keys'), w_keys)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1935,11 +1935,11 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1976,17 +1976,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ListComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2027,17 +2027,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_SetComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2080,19 +2080,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_DictComp)
         w_key = self.key.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('key'), w_key)
+        space.setattr(w_node, space.newtext('key'), w_key)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2137,17 +2137,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_GeneratorExp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2185,11 +2185,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2226,23 +2226,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Compare)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         if self.ops is None:
             ops_w = []
         else:
             ops_w = [cmpop_to_class[node - 1]().to_object(space) for node in self.ops] # cmpop
         w_ops = space.newlist(ops_w)
-        space.setattr(w_node, space.wrap('ops'), w_ops)
+        space.setattr(w_node, space.newtext('ops'), w_ops)
         if self.comparators is None:
             comparators_w = []
         else:
             comparators_w = [node.to_object(space) for node in self.comparators] # expr
         w_comparators = space.newlist(comparators_w)
-        space.setattr(w_node, space.wrap('comparators'), w_comparators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('comparators'), w_comparators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2296,27 +2296,27 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Call)
         w_func = self.func.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('func'), w_func)
+        space.setattr(w_node, space.newtext('func'), w_func)
         if self.args is None:
             args_w = []
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.keywords is None:
             keywords_w = []
         else:
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
-        space.setattr(w_node, space.wrap('keywords'), w_keywords)
+        space.setattr(w_node, space.newtext('keywords'), w_keywords)
         w_starargs = self.starargs.to_object(space) if self.starargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('starargs'), w_starargs)
+        space.setattr(w_node, space.newtext('starargs'), w_starargs)
         w_kwargs = self.kwargs.to_object(space) if self.kwargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('kwargs'), w_kwargs)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2360,11 +2360,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Repr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2397,11 +2397,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Num)
         w_n = self.n  # object
-        space.setattr(w_node, space.wrap('n'), w_n)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('n'), w_n)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2434,11 +2434,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Str)
         w_s = self.s  # string
-        space.setattr(w_node, space.wrap('s'), w_s)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('s'), w_s)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2474,15 +2474,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Attribute)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_attr = space.wrap(self.attr)  # identifier
-        space.setattr(w_node, space.wrap('attr'), w_attr)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_attr = space.newtext(self.attr)  # identifier
+        space.setattr(w_node, space.newtext('attr'), w_attr)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2527,15 +2527,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Subscript)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         w_slice = self.slice.to_object(space)  # slice
-        space.setattr(w_node, space.wrap('slice'), w_slice)
+        space.setattr(w_node, space.newtext('slice'), w_slice)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2576,14 +2576,14 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Name)
-        w_id = space.wrap(self.id)  # identifier
-        space.setattr(w_node, space.wrap('id'), w_id)
+        w_id = space.newtext(self.id)  # identifier
+        space.setattr(w_node, space.newtext('id'), w_id)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2628,13 +2628,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2678,13 +2678,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2720,11 +2720,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Const)
         w_value = self.value  # object
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2866,11 +2866,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
         w_lower = self.lower.to_object(space) if self.lower is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('lower'), w_lower)
+        space.setattr(w_node, space.newtext('lower'), w_lower)
         w_upper = self.upper.to_object(space) if self.upper is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('upper'), w_upper)
+        space.setattr(w_node, space.newtext('upper'), w_upper)
         w_step = self.step.to_object(space) if self.step is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('step'), w_step)
+        space.setattr(w_node, space.newtext('step'), w_step)
         return w_node
 
     @staticmethod
@@ -2907,7 +2907,7 @@
         else:
             dims_w = [node.to_object(space) for node in self.dims] # slice
         w_dims = space.newlist(dims_w)
-        space.setattr(w_node, space.wrap('dims'), w_dims)
+        space.setattr(w_node, space.newtext('dims'), w_dims)
         return w_node
 
     @staticmethod
@@ -2935,7 +2935,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Index)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
@@ -3266,15 +3266,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_comprehension)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.ifs is None:
             ifs_w = []
         else:
             ifs_w = [node.to_object(space) for node in self.ifs] # expr
         w_ifs = space.newlist(ifs_w)
-        space.setattr(w_node, space.wrap('ifs'), w_ifs)
+        space.setattr(w_node, space.newtext('ifs'), w_ifs)
         return w_node
 
     @staticmethod
@@ -3334,19 +3334,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
+        space.setattr(w_node, space.newtext('type'), w_type)
         w_name = self.name.to_object(space) if self.name is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('name'), w_name)
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3394,17 +3394,17 @@
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
-        w_vararg = space.wrap(self.vararg)  # identifier
-        space.setattr(w_node, space.wrap('vararg'), w_vararg)
-        w_kwarg = space.wrap(self.kwarg)  # identifier
-        space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
+        space.setattr(w_node, space.newtext('args'), w_args)
+        w_vararg = space.newtext_or_none(self.vararg)  # identifier
+        space.setattr(w_node, space.newtext('vararg'), w_vararg)
+        w_kwarg = space.newtext_or_none(self.kwarg)  # identifier
+        space.setattr(w_node, space.newtext('kwarg'), w_kwarg)
         if self.defaults is None:
             defaults_w = []
         else:
             defaults_w = [node.to_object(space) for node in self.defaults] # expr
         w_defaults = space.newlist(defaults_w)
-        space.setattr(w_node, space.wrap('defaults'), w_defaults)
+        space.setattr(w_node, space.newtext('defaults'), w_defaults)
         return w_node
 
     @staticmethod
@@ -3438,10 +3438,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.wrap(self.arg)  # identifier
-        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_arg = space.newtext(self.arg)  # identifier
+        space.setattr(w_node, space.newtext('arg'), w_arg)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
@@ -3472,10 +3472,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_alias)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
-        w_asname = space.wrap(self.asname)  # identifier
-        space.setattr(w_node, space.wrap('asname'), w_asname)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
+        w_asname = space.newtext_or_none(self.asname)  # identifier
+        space.setattr(w_node, space.newtext('asname'), w_asname)
         return w_node
 
     @staticmethod
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1070,8 +1070,8 @@
                 raw = "0" + raw
         if negative:
             raw = "-" + raw
-        w_num_str = self.space.wrap(raw)
-        w_base = self.space.wrap(base)
+        w_num_str = self.space.newtext(raw)
+        w_base = self.space.newint(base)
         if raw[-1] in "lL":
             tp = self.space.w_long
             return self.space.call_function(tp, w_num_str, w_base)
@@ -1105,12 +1105,12 @@
                     raise
                 # UnicodeError in literal: turn into SyntaxError
                 e.normalize_exception(space)
-                errmsg = space.str_w(space.str(e.get_w_value(space)))
+                errmsg = space.text_w(space.str(e.get_w_value(space)))
                 raise self.error('(unicode error) %s' % errmsg, atom_node)
             # This implements implicit string concatenation.
             if len(sub_strings_w) > 1:
                 w_sub_strings = space.newlist(sub_strings_w)
-                w_join = space.getattr(space.wrap(""), space.wrap("join"))
+                w_join = space.getattr(space.newtext(""), space.newtext("join"))
                 final_string = space.call_function(w_join, w_sub_strings)
             else:
                 final_string = sub_strings_w[0]
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -322,7 +322,7 @@
     def visit_ClassDef(self, cls):
         self.update_position(cls.lineno, True)
         self.visit_sequence(cls.decorator_list)
-        self.load_const(self.space.wrap(cls.name))
+        self.load_const(self.space.newtext(cls.name))
         self.visit_sequence(cls.bases)
         bases_count = len(cls.bases) if cls.bases is not None else 0
         self.emit_op_arg(ops.BUILD_TUPLE, bases_count)
@@ -611,7 +611,7 @@
                 level = 0
             else:
                 level = -1
-            self.load_const(self.space.wrap(level))
+            self.load_const(self.space.newint(level))
             self.load_const(self.space.w_None)
             self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name)
             # If there's no asname then we store the root module.  If there is
@@ -654,12 +654,12 @@
             level = -1
         else:
             level = imp.level
-        self.load_const(space.wrap(level))
+        self.load_const(space.newint(level))
         names_w = [None]*len(imp.names)
         for i in range(len(imp.names)):
             alias = imp.names[i]
             assert isinstance(alias, ast.alias)
-            names_w[i] = space.wrap(alias.name)
+            names_w[i] = space.newtext(alias.name)
         self.load_const(space.newtuple(names_w))
         if imp.module:
             mod_name = imp.module
@@ -944,7 +944,7 @@
         self.name_op(name.id, name.ctx)
 
     def visit_keyword(self, keyword):
-        self.load_const(self.space.wrap(keyword.arg))
+        self.load_const(self.space.newtext(keyword.arg))
         keyword.value.walkabout(self)
 
     def visit_Call(self, call):
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -19,10 +19,10 @@
 
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
-    w_msg = space.wrap(msg)
-    w_filename = space.wrap(fn)
-    w_lineno = space.wrap(lineno)
-    w_offset = space.wrap(offset)
+    w_msg = space.newtext(msg)
+    w_filename = space.newtext(fn)
+    w_lineno = space.newint(lineno)
+    w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
 
 
@@ -110,9 +110,9 @@
 
 def intern_if_common_string(space, w_const):
     # only intern identifier-like strings
-    if not space.is_w(space.type(w_const), space.w_str):
+    if not space.is_w(space.type(w_const), space.w_text):
         return w_const
-    for c in space.str_w(w_const):
+    for c in space.text_w(w_const):
         if not (c.isalnum() or c == '_'):
             return w_const
     return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -120,7 +120,7 @@
     return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
-    return space.wrap(not space.is_true(operand))
+    return space.newbool(not space.is_true(operand))
 
 
 binary_folders = {
@@ -216,7 +216,7 @@
                         break
                 else:
                     raise AssertionError("unknown unary operation")
-                w_minint = self.space.wrap(-sys.maxint - 1)
+                w_minint = self.space.newint(-sys.maxint - 1)
                 # This makes sure the result is an integer.
                 if self.space.eq_w(w_minint, w_const):
                     w_const = w_minint
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -127,8 +127,14 @@
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
         elif field.type in ("object", "singleton", "string", "bytes"):
             return value
-        elif field.type in ("identifier", "int", "bool"):
-            return "space.wrap(%s)" % (value,)
+        elif field.type == "bool":
+            return "space.newbool(%s)" % (value,)
+        elif field.type == "int":
+            return "space.newint(%s)" % (value,)
+        elif field.type == "identifier":
+            if field.opt:
+                return "space.newtext_or_none(%s)" % (value,)
+            return "space.newtext(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             if field.opt:
@@ -194,7 +200,7 @@
             wrapping_code = self.get_field_converter(field)
             for line in wrapping_code:
                 self.emit(line, 2)
-            self.emit("space.setattr(w_node, space.wrap(%r), w_%s)" % (
+            self.emit("space.setattr(w_node, space.newtext(%r), w_%s)" % (
                     str(field.name), field.name), 2)
         self.emit("return w_node", 2)
         self.emit("")
@@ -402,7 +408,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -437,10 +443,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
+        assert fields == []
 
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -456,14 +462,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -482,12 +488,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -503,7 +509,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -532,16 +538,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)
+            space.newtext(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -5,7 +5,7 @@
 from rpython.rlib import jit, types
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.objectmodel import (we_are_translated, newlist_hint,
-     compute_unique_id, specialize)
+     compute_unique_id, specialize, not_rpython)
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
@@ -51,7 +51,7 @@
         w_dict = self.getdict(space)
         if w_dict is not None:
             try:
-                space.delitem(w_dict, space.wrap(attr))
+                space.delitem(w_dict, space.newtext(attr))
                 return True
             except OperationError as ex:
                 if not ex.match(space, space.w_KeyError):
@@ -77,7 +77,7 @@
 
     def getname(self, space):
         try:
-            return space.str_w(space.getattr(self, space.wrap('__name__')))
+            return space.text_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return '?'
@@ -86,8 +86,8 @@
     def getaddrstring(self, space):
         # slowish
         w_id = space.id(self)
-        w_4 = space.wrap(4)
-        w_0x0F = space.wrap(0x0F)
+        w_4 = space.newint(4)
+        w_0x0F = space.newint(0x0F)
         i = 2 * HUGEVAL_BYTES
         addrstring = [' '] * i
         while True:
@@ -104,8 +104,8 @@
 
     def getrepr(self, space, info, moreinfo=''):
         addrstring = self.getaddrstring(space)
-        return space.wrap("<%s at 0x%s%s>" % (info, addrstring,
-                                              moreinfo))
+        return space.newtext("<%s at 0x%s%s>" % (info, addrstring,
+                                                 moreinfo))
 
     def getslotvalue(self, index):
         raise NotImplementedError
@@ -321,11 +321,14 @@
         raise oefmt(space.w_TypeError,
                     "ord() expected string of length 1, but %T found", self)
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
+        """ Return a version of the object bound to a specific object space
+        instance. This is used for objects (like e.g. TypeDefs) that are
+        constructed before there is an object space instance. """
         return self
 
+    @not_rpython
     def unwrap(self, space):
-        """NOT_RPYTHON"""
         # _____ this code is here to support testing only _____
         return self
 
@@ -408,8 +411,9 @@
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
+    @not_rpython
     def __init__(self, config=None):
-        "NOT_RPYTHON: Basic initialization of objects."
+        "Basic initialization of objects."
         self.fromcache = InternalSpaceCache(self).getorbuild
         self.threadlocals = ThreadLocals()
         # set recursion limit
@@ -487,8 +491,9 @@
         except AttributeError:
             return self.__class__.__name__
 
+    @not_rpython
     def setbuiltinmodule(self, importname):
-        """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules"""
+        """load a lazy pypy/module and put it into sys.modules"""
         if '.' in importname:
             fullname = importname
             importname = fullname.rsplit('.', 1)[1]
@@ -502,13 +507,13 @@
         else:
             name = importname

From pypy.commits at gmail.com  Tue Feb 21 08:14:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 05:14:18 -0800 (PST)
Subject: [pypy-commit] pypy default: Add space.text0_w() and
 space.bytes0_w(). Start using it
Message-ID: <58ac3d2a.56002e0a.d43af.9347@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90263:79bd45d55155
Date: 2017-02-21 14:12 +0100
http://bitbucket.org/pypy/pypy/changeset/79bd45d55155/

Log:	Add space.text0_w() and space.bytes0_w(). Start using it in the imp
	and posix modules.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1622,9 +1622,12 @@
         """For tests only."""
         return self.bytes_w(w_obj)
 
+    #@not_rpython    BACKCOMPAT
+    def str0_w(self, w_obj):
+        return self.bytes0_w(w_obj)
 
-    def str0_w(self, w_obj):
-        "Like str_w, but rejects strings with NUL bytes."
+    def bytes0_w(self, w_obj):
+        "Like bytes_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
         result = w_obj.str_w(self)
         if '\x00' in result:
@@ -1632,6 +1635,10 @@
                         "argument must be a string without NUL characters")
         return rstring.assert_str0(result)
 
+    def text0_w(self, w_obj):
+        "Like text_w, but rejects strings with NUL bytes."
+        return self.bytes0_w(w_obj)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -154,9 +154,15 @@
     def visit_bytes(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_bytes0(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_text(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_text0(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -289,9 +295,15 @@
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
 
+    def visit_bytes0(self, typ):
+        self.run_args.append("space.bytes0_w(%s)" % (self.scopenext(),))
+
     def visit_text(self, typ):
         self.run_args.append("space.text_w(%s)" % (self.scopenext(),))
 
+    def visit_text0(self, typ):
+        self.run_args.append("space.text0_w(%s)" % (self.scopenext(),))
+
     def visit_nonnegint(self, typ):
         self.run_args.append("space.gateway_nonnegint_w(%s)" % (
             self.scopenext(),))
@@ -445,9 +457,15 @@
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
 
+    def visit_bytes0(self, typ):
+        self.unwrap.append("space.bytes0_w(%s)" % (self.nextarg(),))
+
     def visit_text(self, typ):
         self.unwrap.append("space.text_w(%s)" % (self.nextarg(),))
 
+    def visit_text0(self, typ):
+        self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),))
+
     def visit_nonnegint(self, typ):
         self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -172,7 +172,7 @@
     ctxt_package = None
     if ctxt_w_package is not None and ctxt_w_package is not space.w_None:
         try:
-            ctxt_package = space.str0_w(ctxt_w_package)
+            ctxt_package = space.text0_w(ctxt_w_package)
         except OperationError as e:
             if not e.match(space, space.w_TypeError):
                 raise
@@ -220,7 +220,7 @@
         ctxt_name = None
         if ctxt_w_name is not None:
             try:
-                ctxt_name = space.str0_w(ctxt_w_name)
+                ctxt_name = space.text0_w(ctxt_w_name)
             except OperationError as e:
                 if not e.match(space, space.w_TypeError):
                     raise
@@ -263,7 +263,7 @@
     return rel_modulename, rel_level
 
 
- at unwrap_spec(name='str0', level=int)
+ at unwrap_spec(name='text0', level=int)
 def importhook(space, name, w_globals=None,
                w_locals=None, w_fromlist=None, level=-1):
     modulename = name
@@ -431,7 +431,7 @@
                 for i in range(length):
                     w_name = space.getitem(w_fromlist, space.newint(i))
                     if try_getattr(space, w_mod, w_name) is None:
-                        load_part(space, w_path, prefix, space.str0_w(w_name),
+                        load_part(space, w_path, prefix, space.text0_w(w_name),
                                   w_mod, tentative=1)
         return w_mod
     else:
@@ -728,7 +728,7 @@
         raise oefmt(space.w_TypeError, "reload() argument must be module")
 
     w_modulename = space.getattr(w_module, space.newtext("__name__"))
-    modulename = space.str0_w(w_modulename)
+    modulename = space.text0_w(w_modulename)
     if not space.is_w(check_sys_modules(space, w_modulename), w_module):
         raise oefmt(space.w_ImportError,
                     "reload(): module %s not in sys.modules", modulename)
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -49,7 +49,7 @@
         return space.interp_w(W_File, w_file).stream
 
 def find_module(space, w_name, w_path=None):
-    name = space.str0_w(w_name)
+    name = space.text0_w(w_name)
     if space.is_none(w_path):
         w_path = None
 
@@ -143,7 +143,7 @@
     return Module(space, w_name, add_package=False)
 
 def init_builtin(space, w_name):
-    name = space.str0_w(w_name)
+    name = space.text0_w(w_name)
     if name not in space.builtin_modules:
         return
     if space.finditem(space.sys.get('modules'), w_name) is not None:
@@ -155,7 +155,7 @@
     return None
 
 def is_builtin(space, w_name):
-    name = space.str0_w(w_name)
+    name = space.text0_w(w_name)
     if name not in space.builtin_modules:
         return space.newint(0)
     if space.finditem(space.sys.get('modules'), w_name) is not None:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -38,9 +38,9 @@
 
 def fsencode_w(space, w_obj):
     if space.isinstance_w(w_obj, space.w_unicode):
-        w_obj = space.call_method(w_obj, 'encode',
+        w_obj = space.call_method(space.w_unicode, 'encode', w_obj,
                                   getfilesystemencoding(space))
-    return space.str0_w(w_obj)
+    return space.bytes0_w(w_obj)
 
 class FileEncoder(object):
     is_unicode = True
@@ -63,7 +63,7 @@
         self.w_obj = w_obj
 
     def as_bytes(self):
-        return self.space.str0_w(self.w_obj)
+        return self.space.bytes0_w(self.w_obj)
 
     def as_unicode(self):
         space = self.space
@@ -78,7 +78,7 @@
             fname = FileEncoder(space, w_fname)
             return func(fname, *args)
         else:
-            fname = space.str0_w(w_fname)
+            fname = space.bytes0_w(w_fname)
             return func(fname, *args)
     return dispatch
 
@@ -398,7 +398,7 @@
                                space.newfloat(times[3]),
                                space.newfloat(times[4])])
 
- at unwrap_spec(cmd='str0')
+ at unwrap_spec(cmd='text0')
 def system(space, cmd):
     """Execute the command (a string) in a subshell."""
     try:
@@ -430,7 +430,7 @@
             fullpath = rposix.getfullpathname(path)
             w_fullpath = space.newunicode(fullpath)
         else:
-            path = space.str0_w(w_path)
+            path = space.bytes0_w(w_path)
             fullpath = rposix.getfullpathname(path)
             w_fullpath = space.newbytes(fullpath)
     except OSError as e:
@@ -534,7 +534,7 @@
     for key, value in os.environ.items():
         space.setitem(w_env, space.newtext(key), space.newtext(value))
 
- at unwrap_spec(name='str0', value='str0')
+ at unwrap_spec(name='text0', value='text0')
 def putenv(space, name, value):
     """Change or add an environment variable."""
     if _WIN32 and len(name) > _MAX_ENV:
@@ -554,7 +554,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(name='str0')
+ at unwrap_spec(name='text0')
 def unsetenv(space, name):
     """Delete an environment variable."""
     try:
@@ -600,7 +600,7 @@
                 result_w[i] = w_res
             return space.newlist(result_w)
         else:
-            dirname = space.str0_w(w_dirname)
+            dirname = space.bytes0_w(w_dirname)
             result = rposix.listdir(dirname)
             # The list comprehension is a workaround for an obscure translation
             # bug.
@@ -820,7 +820,7 @@
     w_keys = space.call_method(w_env, 'keys')
     for w_key in space.unpackiterable(w_keys):
         w_value = space.getitem(w_env, w_key)
-        env[space.str0_w(w_key)] = space.str0_w(w_value)
+        env[space.text0_w(w_key)] = space.text0_w(w_value)
     return env
 
 def execve(space, w_command, w_args, w_env):

From pypy.commits at gmail.com  Tue Feb 21 08:33:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 05:33:32 -0800 (PST)
Subject: [pypy-commit] pypy default: Add space.newfilename(),
 which is meant to replace the py3.5
Message-ID: <58ac41ac.501d190a.fc809.909f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90264:168885e476f6
Date: 2017-02-21 14:32 +0100
http://bitbucket.org/pypy/pypy/changeset/168885e476f6/

Log:	Add space.newfilename(), which is meant to replace the py3.5
	space.wrap_fsdecoded()

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -501,7 +501,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.newtext(cur)
+        return space.newfilename(cur)
 
 # ____________________________________________________________
 
@@ -915,7 +915,7 @@
         r = os.uname()
     except OSError as e:
         raise wrap_oserror(space, e)
-    l_w = [space.newtext(i) for i in [r[0], r[1], r[2], r[3], r[4]]]
+    l_w = [space.newfilename(i) for i in [r[0], r[1], r[2], r[3], r[4]]]
     return space.newtuple(l_w)
 
 def getuid(space):
@@ -1226,7 +1226,7 @@
 @unwrap_spec(fd=c_int)
 def ttyname(space, fd):
     try:
-        return space.newtext(os.ttyname(fd))
+        return space.newfilename(os.ttyname(fd))
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -1365,4 +1365,4 @@
 
     Return the name of the controlling terminal for this process.
     """
-    return space.newtext(os.ctermid())
+    return space.newfilename(os.ctermid())
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -213,6 +213,7 @@
 
     newtext = newbytes
     newtext_or_none = newbytes
+    newfilename = newbytes
 
     @not_rpython
     def wrap(self, x):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -343,6 +343,10 @@
             return self.w_None
         return self.newtext(s)
 
+    def newfilename(self, s):
+        assert isinstance(s, str) # on pypy3, this decodes the byte string
+        return W_BytesObject(s)   # with the filesystem encoding
+
     def newunicode(self, uni):
         assert uni is not None
         assert isinstance(uni, unicode)

From pypy.commits at gmail.com  Tue Feb 21 08:42:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 05:42:15 -0800 (PST)
Subject: [pypy-commit] pypy default: Move fsencode_w() as a space method,
 and add a 'fsencode' unwrap_spec.
Message-ID: <58ac43b7.924d2e0a.54625.9018@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90265:be3d0e6c4201
Date: 2017-02-21 14:41 +0100
http://bitbucket.org/pypy/pypy/changeset/be3d0e6c4201/

Log:	Move fsencode_w() as a space method, and add a 'fsencode'
	unwrap_spec.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1639,6 +1639,14 @@
         "Like text_w, but rejects strings with NUL bytes."
         return self.bytes0_w(w_obj)
 
+    def fsencode_w(self, w_obj):
+        "Like text0_w, but unicodes are encoded with the filesystem encoding."
+        if self.isinstance_w(w_obj, self.w_unicode):
+            from pypy.module.sys.interp_encoding import getfilesystemencoding
+            w_obj = self.call_method(self.w_unicode, 'encode', w_obj,
+                                     getfilesystemencoding(self))
+        return self.bytes0_w(w_obj)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -163,6 +163,9 @@
     def visit_text0(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_fsencode(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -304,6 +307,9 @@
     def visit_text0(self, typ):
         self.run_args.append("space.text0_w(%s)" % (self.scopenext(),))
 
+    def visit_fsencode(self, typ):
+        self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
+
     def visit_nonnegint(self, typ):
         self.run_args.append("space.gateway_nonnegint_w(%s)" % (
             self.scopenext(),))
@@ -466,6 +472,9 @@
     def visit_text0(self, typ):
         self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),))
 
+    def visit_fsencode(self, typ):
+        self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
+
     def visit_nonnegint(self, typ):
         self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -36,12 +36,6 @@
         return space.newint(uid)     # an unsigned number
 wrap_gid = wrap_uid
 
-def fsencode_w(space, w_obj):
-    if space.isinstance_w(w_obj, space.w_unicode):
-        w_obj = space.call_method(space.w_unicode, 'encode', w_obj,
-                                  getfilesystemencoding(space))
-    return space.bytes0_w(w_obj)
-
 class FileEncoder(object):
     is_unicode = True
 
@@ -50,7 +44,7 @@
         self.w_obj = w_obj
 
     def as_bytes(self):
-        return fsencode_w(self.space, self.w_obj)
+        return self.space.fsencode_w(self.w_obj)
 
     def as_unicode(self):
         return self.space.unicode0_w(self.w_obj)
@@ -698,10 +692,9 @@
     import signal
     rposix.kill(os.getpid(), signal.SIGABRT)
 
-def link(space, w_src, w_dst):
+ at unwrap_spec(src='fsencode', dst='fsencode')
+def link(space, src, dst):
     "Create a hard link to a file."
-    src = fsencode_w(space, w_src)
-    dst = fsencode_w(space, w_dst)
     try:
         os.link(src, dst)
     except OSError as e:
@@ -714,9 +707,9 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
-def readlink(space, w_path):
+ at unwrap_spec(path='fsencode')
+def readlink(space, path):
     "Return a string representing the path to which the symbolic link points."
-    path = fsencode_w(space, w_path)
     try:
         result = os.readlink(path)
     except OSError as e:
@@ -805,7 +798,8 @@
 def _exit(space, status):
     os._exit(status)
 
-def execv(space, w_command, w_args):
+ at unwrap_spec(command='fsencode')
+def execv(space, command, w_args):
     """ execv(path, args)
 
 Execute an executable path with arguments, replacing current process.
@@ -813,7 +807,7 @@
         path: path of executable file
         args: iterable of strings
     """
-    execve(space, w_command, w_args, None)
+    execve(space, command, w_args, None)
 
 def _env2interp(space, w_env):
     env = {}
@@ -823,7 +817,8 @@
         env[space.text0_w(w_key)] = space.text0_w(w_value)
     return env
 
-def execve(space, w_command, w_args, w_env):
+ at unwrap_spec(command='fsencode')
+def execve(space, command, w_args, w_env):
     """ execve(path, args, env)
 
 Execute a path with arguments and environment, replacing current process.
@@ -832,13 +827,12 @@
         args: iterable of arguments
         env: dictionary of strings mapping to strings
     """
-    command = fsencode_w(space, w_command)
     try:
         args_w = space.unpackiterable(w_args)
         if len(args_w) < 1:
             raise oefmt(space.w_ValueError,
                         "execv() must have at least one argument")
-        args = [fsencode_w(space, w_arg) for w_arg in args_w]
+        args = [space.fsencode_w(w_arg) for w_arg in args_w]
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
@@ -857,20 +851,18 @@
         except OSError as e:
             raise wrap_oserror(space, e)
 
- at unwrap_spec(mode=int)
-def spawnv(space, mode, w_path, w_args):
-    path = fsencode_w(space, w_path)
-    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
+ at unwrap_spec(mode=int, path='fsencode')
+def spawnv(space, mode, path, w_args):
+    args = [space.fsencode_w(w_arg) for w_arg in space.unpackiterable(w_args)]
     try:
         ret = os.spawnv(mode, path, args)
     except OSError as e:
         raise wrap_oserror(space, e)
     return space.newint(ret)
 
- at unwrap_spec(mode=int)
-def spawnve(space, mode, w_path, w_args, w_env):
-    path = fsencode_w(space, w_path)
-    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
+ at unwrap_spec(mode=int, path='fsencode')
+def spawnve(space, mode, path, w_args, w_env):
+    args = [space.fsencode_w(w_arg) for w_arg in space.unpackiterable(w_args)]
     env = _env2interp(space, w_env)
     try:
         ret = os.spawnve(mode, path, args, env)
@@ -969,12 +961,12 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
-def chroot(space, w_path):
+ at unwrap_spec(path='fsencode')
+def chroot(space, path):
     """ chroot(path)
 
     Change root directory to path.
     """
-    path = fsencode_w(space, w_path)
     try:
         os.chroot(path)
     except OSError as e:
@@ -1259,8 +1251,8 @@
         raise wrap_oserror(space, e)
     return space.newint(res)
 
-def pathconf(space, w_path, w_name):
-    path = fsencode_w(space, w_path)
+ at unwrap_spec(path='fsencode')
+def pathconf(space, path, w_name):
     num = confname_w(space, w_name, os.pathconf_names)
     try:
         res = os.pathconf(path, num)
@@ -1276,20 +1268,18 @@
         raise wrap_oserror(space, e)
     return space.newtext(res)
 
- at unwrap_spec(uid=c_uid_t, gid=c_gid_t)
-def chown(space, w_path, uid, gid):
+ at unwrap_spec(path='fsencode', uid=c_uid_t, gid=c_gid_t)
+def chown(space, path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid."""
-    path = fsencode_w(space, w_path)
     try:
         os.chown(path, uid, gid)
     except OSError as e:
         raise wrap_oserror(space, e, path)
 
- at unwrap_spec(uid=c_uid_t, gid=c_gid_t)
-def lchown(space, w_path, uid, gid):
+ at unwrap_spec(path='fsencode', uid=c_uid_t, gid=c_gid_t)
+def lchown(space, path, uid, gid):
     """Change the owner and group id of path to the numeric uid and gid.
 This function will not follow symbolic links."""
-    path = fsencode_w(space, w_path)
     try:
         os.lchown(path, uid, gid)
     except OSError as e:

From pypy.commits at gmail.com  Tue Feb 21 08:56:57 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Feb 2017 05:56:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: pickle restriction for memoryview,
 also do not pickle objects that do not have a __new__ type method
Message-ID: <58ac4729.0e152e0a.20484.93f7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90266:91b1933d24b7
Date: 2017-02-21 14:50 +0100
http://bitbucket.org/pypy/pypy/changeset/91b1933d24b7/

Log:	pickle restriction for memoryview, also do not pickle objects that
	do not have a __new__ type method

diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -471,7 +471,7 @@
         assert 'finished' in repr(y)
         assert y.gi_code is None
 
-    def test_memoryview(self):
+    def test_pickle_memoryview(self):
         import pickle
         raises(TypeError, pickle.dumps, memoryview(b"abc"))
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -689,6 +689,9 @@
             bytes = self.as_str()
             return _array_to_hexstring(space, StringBuffer(bytes), 0, 1, len(bytes))
 
+    def descr___getstate__(self, space):
+        raise oefmt(space.w_TypeError, "cannot pickle memoryview objects")
+
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'
 
@@ -711,6 +714,7 @@
     __enter__   = interp2app(W_MemoryView.descr_enter),
     __exit__    = interp2app(W_MemoryView.descr_exit),
     __weakref__ = make_weakref_descr(W_MemoryView),
+    __getstate__= interp2app(W_MemoryView.descr___getstate__),
     cast        = interp2app(W_MemoryView.descr_cast),
     hex         = interp2app(W_MemoryView.descr_hex),
     tobytes     = interp2app(W_MemoryView.descr_tobytes),
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -24,6 +24,9 @@
     try:
         getstate = obj.__getstate__
     except AttributeError:
+        # TODO restrict pickling of variable sized objects
+        # tp_itemsize != 0, for now e.g. memoryview is handled
+        # by raising a TypeError in __getstate__
         state = getattr(obj, "__dict__", None)
         names = slotnames(cls) # not checking for list
         if names is not None:
@@ -44,6 +47,9 @@
 def reduce_2(obj, proto, args, kwargs):
     cls = obj.__class__
 
+    if not hasattr(type(obj), "__new__"):
+        raise TypeError("can't pickle %s objects" % type(obj).__name__)
+
     import copyreg
 
     if not isinstance(args, tuple):
@@ -58,7 +64,6 @@
        raise ValueError("must use protocol 4 or greater to copy this "
                         "object; since __getnewargs_ex__ returned "
                         "keyword arguments.")
-
     state = _getstate(obj)
     listitems = iter(obj) if isinstance(obj, list) else None
     dictitems = iter(obj.items()) if isinstance(obj, dict) else None

From pypy.commits at gmail.com  Tue Feb 21 09:13:00 2017
From: pypy.commits at gmail.com (fijal)
Date: Tue, 21 Feb 2017 06:13:00 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: start testing using hypothesis
Message-ID: <58ac4aec.cdd8190a.f6b38.9d25@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90267:422b66748b74
Date: 2017-02-21 15:13 +0100
http://bitbucket.org/pypy/pypy/changeset/422b66748b74/

Log:	start testing using hypothesis

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -60,10 +60,10 @@
             xxx
             substr = decode_unicode_utf8(space, s, ps, q)
         if rawmode:
-            v = unicodehelper.decode_raw_unicode_escape(space, substr)
+            v, length = unicodehelper.decode_raw_unicode_escape(space, substr)
         else:
-            v = unicodehelper.decode_unicode_escape(space, substr)
-        return space.newunicode(v)
+            v, length = unicodehelper.decode_unicode_escape(space, substr)
+        return space.newunicode(v, length)
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "utf8" and
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -39,18 +39,18 @@
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
     # XXX pick better length, maybe
-    result, consumed = rutf8.str_decode_utf8_escape(
+    result, consumed, length = rutf8.str_decode_utf8_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         unicodedata_handler=unicodedata_handler)
-    return result
+    return result, length
 
 def decode_raw_unicode_escape(space, string):
     # XXX pick better length, maybe
-    result, consumed = rutf8.str_decode_raw_utf8_escape(
+    result, consumed, length = rutf8.str_decode_raw_utf8_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space))
-    return result
+    return result, length
 
 def decode_utf8(space, string):
     # Surrogates are accepted and not treated specially at all.
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -398,7 +398,7 @@
 
 @marshaller(W_UnicodeObject)
 def marshal_unicode(space, w_unicode, m):
-    s = unicodehelper.encode_utf8(space, space.unicode_w(w_unicode))
+    s = space.utf8_w(w_unicode)
     m.atom_str(TYPE_UNICODE, s)
 
 @unmarshaller(TYPE_UNICODE)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -346,10 +346,10 @@
             return self.w_None
         return self.newtext(s)
 
-    def newunicode(self, utf8s):
+    def newunicode(self, utf8s, length):
         assert utf8s is not None
         assert isinstance(utf8s, str)
-        return W_UnicodeObject(utf8s)
+        return W_UnicodeObject(utf8s, length)
 
     def type(self, w_obj):
         jit.promote(w_obj.__class__)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -32,11 +32,12 @@
     _immutable_fields_ = ['_utf8']
 
     @enforceargs(utf8str=str)
-    def __init__(self, utf8str, ucs4str=None):
+    def __init__(self, utf8str, length, ucs4str=None):
         assert isinstance(utf8str, str)
         if ucs4str is not None:
             assert isinstance(ucs4str, unicode)
         self._utf8 = utf8str
+        self._length = length
         self._ucs4 = ucs4str
 
     def __repr__(self):
@@ -508,14 +509,13 @@
         if encoding == 'ascii':
             # XXX error handling
             s = space.charbuf_w(w_obj)
-            xxx
             try:
-                u = fast_str_decode_ascii(s)
-            except ValueError:
-                eh = unicodehelper.decode_error_handler(space)
-                u = str_decode_ascii(     # try again, to get the error right
-                    s, len(s), None, final=True, errorhandler=eh)[0]
-            return space.newunicode(u)
+                rutf8.check_ascii(s)
+            except rutf8.AsciiCheckError as e:
+                unicodehelper.decode_error_handler(space)(None,
+                    'ascii', "ordinal not in range(128)", s, e.pos, e.pos+1)
+                assert False
+            return space.newunicode(s)
         if encoding == 'utf-8':
             yyy
             s = space.charbuf_w(w_obj)
@@ -1130,7 +1130,7 @@
     return [s for s in value]
 
 
-W_UnicodeObject.EMPTY = W_UnicodeObject('')
+W_UnicodeObject.EMPTY = W_UnicodeObject('', 0)
 
 
 # Helper for converting int/long
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -24,19 +24,21 @@
     raise ValueError
 
 class AsciiCheckError(Exception):
-    pass
+    def __init__(self, pos):
+        self.pos = pos
 
 def check_ascii(s):
     for i in range(0, len(s)):
         if ord(s[i]) & 0x80:
-            raise AsciiCheckError
+            raise AsciiCheckError(i)
 
 def str_decode_raw_utf8_escape(s, size, errors, final=False,
                                errorhandler=None):
+    lgt = 0
     if errorhandler is None:
         errorhandler = None # default_unicode_error_decode
     if size == 0:
-        return '', 0
+        return '', 0, 0
     result = StringBuilder(size)
     pos = 0
     while pos < size:
@@ -46,6 +48,7 @@
         if ch != '\\':
             result.append(ch)
             pos += 1
+            lgt += 1
             continue
 
         # \u-escapes are only interpreted iff the number of leading
@@ -55,10 +58,12 @@
             pos += 1
             if pos == size or s[pos] != '\\':
                 break
+            lgt += 1
             result.append('\\')
 
         # we have a backslash at the end of the string, stop here
         if pos >= size:
+            lgt += 1
             result.append('\\')
             break
 
@@ -67,6 +72,7 @@
             (s[pos] != 'u' and s[pos] != 'U')):
             result.append('\\')
             result.append(s[pos])
+            lgt += 2
             pos += 1
             continue
 
@@ -77,7 +83,7 @@
         pos = hexescape(result, s, pos, digits,
                         "rawunicodeescape", errorhandler, message, errors)
 
-    return result.build(), pos
+    return result.build(), pos, lgt
 
 def str_decode_utf8_escape(s, size, errors, final=False,
                               errorhandler=None,
@@ -88,6 +94,7 @@
     if size == 0:
         return '', 0
 
+    lgt = 0
     builder = StringBuilder(size)
     pos = 0
     while pos < size:
@@ -97,6 +104,7 @@
         if ch != '\\':
             builder.append(ch)
             pos += 1
+            lgt += 1
             continue
 
         # - Escapes
@@ -106,22 +114,23 @@
             res, pos = errorhandler(errors, "unicodeescape",
                                     message, s, pos-1, size)
             builder.append(res)
+            lgt += 1
             continue
 
         ch = s[pos]
         pos += 1
         # \x escapes
         if ch == '\n': pass
-        elif ch == '\\': builder.append('\\')
-        elif ch == '\'': builder.append('\'')
-        elif ch == '\"': builder.append('\"')
-        elif ch == 'b' : builder.append('\b')
-        elif ch == 'f' : builder.append('\f')
-        elif ch == 't' : builder.append('\t')
-        elif ch == 'n' : builder.append('\n')
-        elif ch == 'r' : builder.append('\r')
-        elif ch == 'v' : builder.append('\v')
-        elif ch == 'a' : builder.append('\a')
+        elif ch == '\\': builder.append('\\'); lgt += 1
+        elif ch == '\'': builder.append('\''); lgt += 1
+        elif ch == '\"': builder.append('\"'); lgt += 1
+        elif ch == 'b' : builder.append('\b'); lgt += 1
+        elif ch == 'f' : builder.append('\f'); lgt += 1
+        elif ch == 't' : builder.append('\t'); lgt += 1
+        elif ch == 'n' : builder.append('\n'); lgt += 1
+        elif ch == 'r' : builder.append('\r'); lgt += 1
+        elif ch == 'v' : builder.append('\v'); lgt += 1
+        elif ch == 'a' : builder.append('\a'); lgt += 1
         elif '0' <= ch <= '7':
             xxx
             x = ord(ch) - ord('0')
@@ -199,5 +208,6 @@
         else:
             builder.append('\\')
             builder.append(ch)
+            lgt += 2
 
-    return builder.build(), pos
+    return builder.build(), pos, lgt
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rutf8.py
@@ -0,0 +1,28 @@
+
+from hypothesis import given, strategies, settings
+
+from rpython.rlib import rutf8, runicode
+
+ at given(strategies.integers(min_value=0, max_value=runicode.MAXUNICODE))
+def test_unichr_as_utf8(i):
+    assert rutf8.unichr_as_utf8(i) == runicode.UNICHR(i).encode('utf8')
+
+ at given(strategies.binary())
+def test_check_ascii(s):
+    raised = False
+    try:
+        s.decode('ascii')
+    except UnicodeDecodeError as e:
+        raised = True
+    try:
+        rutf8.check_ascii(s)
+    except rutf8.AsciiCheckError as a:
+        assert raised
+        assert a.pos == e.start
+    else:
+        assert not raised
+
+ at given(strategies.binary())
+def test_str_decode_raw_utf8_escape(uni):
+    return # XXX fix details
+    rutf8.str_decode_raw_utf8_escape(uni, len(uni), None)
\ No newline at end of file

From pypy.commits at gmail.com  Tue Feb 21 09:21:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 06:21:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58ac4cfe.5591190a.86c8b.997f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90268:6cb34cc6c598
Date: 2017-02-21 15:20 +0100
http://bitbucket.org/pypy/pypy/changeset/6cb34cc6c598/

Log:	hg merge default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1602,6 +1602,7 @@
     def bytes_w(self, w_obj):
         return w_obj.bytes_w(self)
 
+    #@not_rpython    BACKCOMPAT
     def str0_w(self, w_obj):
         "Like str_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
@@ -1620,6 +1621,15 @@
                         "argument must be a string without NUL characters")
         return rstring.assert_str0(result)
 
+    def text0_w(self, w_obj):
+        "Like text_w, but rejects strings with NUL bytes."
+        from rpython.rlib import rstring
+        result = self.text_w(w_obj)
+        if '\x00' in result:
+            raise oefmt(self.w_ValueError,
+                        "argument must be a string without NUL characters")
+        return rstring.assert_str0(result)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1717,8 +1727,9 @@
             w_obj = self.fsdecode(w_obj)
         return self.unicode0_w(w_obj)
 
+    # BACKCOMPAT  -- replace me with newfilename()
     def wrap_fsdecoded(self, x):
-        return self.fsdecode(self.newbytes(x))
+        return self.newfilename(x)
 
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -168,9 +168,15 @@
     def visit_bytes(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_bytes0(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_text(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_text0(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_fsencode(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -314,9 +320,15 @@
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
 
+    def visit_bytes0(self, typ):
+        self.run_args.append("space.bytes0_w(%s)" % (self.scopenext(),))
+
     def visit_text(self, typ):
         self.run_args.append("space.text_w(%s)" % (self.scopenext(),))
 
+    def visit_text0(self, typ):
+        self.run_args.append("space.text0_w(%s)" % (self.scopenext(),))
+
     def visit_fsencode(self, typ):
         self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
 
@@ -479,9 +491,15 @@
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
 
+    def visit_bytes0(self, typ):
+        self.unwrap.append("space.bytes0_w(%s)" % (self.nextarg(),))
+
     def visit_text(self, typ):
         self.unwrap.append("space.text_w(%s)" % (self.nextarg(),))
 
+    def visit_text0(self, typ):
+        self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),))
+
     def visit_fsencode(self, typ):
         self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -33,6 +33,11 @@
     def getallvalues(self):
         return {0: self._value}
 
+    def _cleanup_(self):
+        # should still be unfilled at this point during translation.
+        # but in some corner cases it is not...  unsure why
+        self._value = None
+
 
 def make_weak_value_dictionary(space, keytype, valuetype):
     "NOT_RPYTHON"
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -286,6 +286,8 @@
     else:
         assert ec.cpyext_gilstate_counter_noleave == 0
         assert oldstate == PyGILState_UNLOCKED
+        assert space.config.translation.thread
+        #      ^^^ otherwise, we should not reach this case
         ec.cpyext_threadstate_is_current = False
         space.threadlocals.leave_thread(space)
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -67,7 +67,7 @@
 lib_pypy = os.path.join(os.path.dirname(__file__),
                         '..', '..', '..', 'lib_pypy')
 
- at unwrap_spec(modulename='str0', level=int)
+ at unwrap_spec(modulename='text0', level=int)
 def importhook(space, modulename, w_globals=None, w_locals=None, w_fromlist=None, level=0):
     # A minimal version, that can only import builtin and lib_pypy modules!
     assert w_locals is w_globals
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -72,7 +72,7 @@
     return None
 
 def is_builtin(space, w_name):
-    name = space.str0_w(w_name)
+    name = space.text0_w(w_name)
     if name not in space.builtin_modules:
         return space.newint(0)
     if space.finditem(space.sys.get('modules'), w_name) is not None:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -66,7 +66,7 @@
         self.w_obj = w_obj
 
     def as_bytes(self):
-        return self.space.fsencode_w(self.w_obj)
+        return self.space.bytes0_w(self.w_obj)
 
     def as_unicode(self):
         return self.space.fsdecode_w(self.w_obj)
@@ -85,7 +85,7 @@
             fname = FileEncoder(space, w_fname)
             return func(fname, *args)
         else:
-            fname = space.fsencode_w(w_fname)
+            fname = space.bytes0_w(w_fname)
             return func(fname, *args)
     return dispatch
 
@@ -697,7 +697,7 @@
             fullpath = rposix.getfullpathname(path)
             w_fullpath = space.newunicode(fullpath)
         else:
-            path = space.str0_w(w_path)
+            path = space.bytes0_w(w_path)
             fullpath = rposix.getfullpathname(path)
             w_fullpath = space.newbytes(fullpath)
     except OSError as e:
@@ -793,7 +793,7 @@
         cur = os.getlogin()
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    return space.wrap_fsdecoded(cur)
+    return space.newfilename(cur)
 
 # ____________________________________________________________
 
@@ -884,7 +884,7 @@
     if space.is_none(w_path):
         w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
-        dirname = space.str0_w(w_path)
+        dirname = space.bytes0_w(w_path)
         try:
             result = rposix.listdir(dirname)
         except OSError as e:
@@ -915,7 +915,7 @@
         if _WIN32:
             result_w[i] = space.newunicode(result[i])
         else:
-            result_w[i] = space.wrap_fsdecoded(result[i])
+            result_w[i] = space.newfilename(result[i])
     return space.newlist(result_w)
 
 @unwrap_spec(fd=c_int)
@@ -1607,7 +1607,7 @@
         r = os.uname()
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
-    l_w = [space.wrap_fsdecoded(i)
+    l_w = [space.newfilename(i)
            for i in [r[0], r[1], r[2], r[3], r[4]]]
     w_tuple = space.newtuple(l_w)
     w_uname_result = space.getattr(space.getbuiltinmodule(os.name),
@@ -1945,7 +1945,7 @@
 @unwrap_spec(fd=c_int)
 def ttyname(space, fd):
     try:
-        return space.wrap_fsdecoded(os.ttyname(fd))
+        return space.newfilename(os.ttyname(fd))
     except OSError as e:
         raise wrap_oserror(space, e, eintr_retry=False)
 
@@ -2156,7 +2156,7 @@
 
     Return the name of the controlling terminal for this process.
     """
-    return space.wrap_fsdecoded(os.ctermid())
+    return space.newfilename(os.ctermid())
 
 @unwrap_spec(fd=c_int)
 def device_encoding(space, fd):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -217,11 +217,10 @@
     def newunicode(self, x):
         return w_some_obj()
 
-    def newutf8(self, x):
+    def newtext(self, x):
         return w_some_obj()
-
-    newtext = newutf8
-    newtext_or_none = newutf8
+    newtext_or_none = newtext
+    newfilename = newtext
 
     @not_rpython
     def wrap(self, x):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -378,6 +378,9 @@
             return self.w_None
         return self.newtext(s)
 
+    def newfilename(self, s):
+        return self.fsdecode(self.newbytes(s))
+
     def newunicode(self, uni):
         assert uni is not None
         assert isinstance(uni, unicode)

From pypy.commits at gmail.com  Tue Feb 21 09:21:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 06:21:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <58ac4d00.e30f190a.19f1e.8ef0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90269:23a4859e0e29
Date: 2017-02-21 15:21 +0100
http://bitbucket.org/pypy/pypy/changeset/23a4859e0e29/

Log:	merge heads

diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -471,7 +471,7 @@
         assert 'finished' in repr(y)
         assert y.gi_code is None
 
-    def test_memoryview(self):
+    def test_pickle_memoryview(self):
         import pickle
         raises(TypeError, pickle.dumps, memoryview(b"abc"))
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -689,6 +689,9 @@
             bytes = self.as_str()
             return _array_to_hexstring(space, StringBuffer(bytes), 0, 1, len(bytes))
 
+    def descr___getstate__(self, space):
+        raise oefmt(space.w_TypeError, "cannot pickle memoryview objects")
+
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'
 
@@ -711,6 +714,7 @@
     __enter__   = interp2app(W_MemoryView.descr_enter),
     __exit__    = interp2app(W_MemoryView.descr_exit),
     __weakref__ = make_weakref_descr(W_MemoryView),
+    __getstate__= interp2app(W_MemoryView.descr___getstate__),
     cast        = interp2app(W_MemoryView.descr_cast),
     hex         = interp2app(W_MemoryView.descr_hex),
     tobytes     = interp2app(W_MemoryView.descr_tobytes),
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -24,6 +24,9 @@
     try:
         getstate = obj.__getstate__
     except AttributeError:
+        # TODO restrict pickling of variable sized objects
+        # tp_itemsize != 0, for now e.g. memoryview is handled
+        # by raising a TypeError in __getstate__
         state = getattr(obj, "__dict__", None)
         names = slotnames(cls) # not checking for list
         if names is not None:
@@ -44,6 +47,9 @@
 def reduce_2(obj, proto, args, kwargs):
     cls = obj.__class__
 
+    if not hasattr(type(obj), "__new__"):
+        raise TypeError("can't pickle %s objects" % type(obj).__name__)
+
     import copyreg
 
     if not isinstance(args, tuple):
@@ -58,7 +64,6 @@
        raise ValueError("must use protocol 4 or greater to copy this "
                         "object; since __getnewargs_ex__ returned "
                         "keyword arguments.")
-
     state = _getstate(obj)
     listitems = iter(obj) if isinstance(obj, list) else None
     dictitems = iter(obj.items()) if isinstance(obj, dict) else None

From pypy.commits at gmail.com  Tue Feb 21 09:40:57 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Feb 2017 06:40:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add variable_sized property to TypeDef
 (similar to the tp_itemsize property in cpython),
 use it to filter types from pickle properly (partly reverts the last commit)
Message-ID: <58ac5179.43502e0a.b5243.8fc5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90270:3a002d0cdb81
Date: 2017-02-21 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/3a002d0cdb81/

Log:	add variable_sized property to TypeDef (similar to the tp_itemsize
	property in cpython), use it to filter types from pickle properly
	(partly reverts the last commit)

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -14,7 +14,8 @@
 
 class TypeDef(object):
     def __init__(self, __name, __base=None, __total_ordering__=None,
-                 __buffer=None, __confirm_applevel_del__=False, **rawdict):
+                 __buffer=None, __confirm_applevel_del__=False,
+                 variable_sized=False, **rawdict):
         "NOT_RPYTHON: initialization-time only"
         self.name = __name
         if __base is None:
@@ -44,6 +45,7 @@
         assert __total_ordering__ in (None, 'auto'), "Unknown value for __total_ordering"
         if __total_ordering__ == 'auto':
             self.auto_total_ordering()
+        self.variable_sized = variable_sized
 
     def add_entries(self, **rawdict):
         # xxx fix the names of the methods to match what app-level expects
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -689,9 +689,6 @@
             bytes = self.as_str()
             return _array_to_hexstring(space, StringBuffer(bytes), 0, 1, len(bytes))
 
-    def descr___getstate__(self, space):
-        raise oefmt(space.w_TypeError, "cannot pickle memoryview objects")
-
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'
 
@@ -699,7 +696,7 @@
     return flags & (MEMORYVIEW_SCALAR|MEMORYVIEW_C)
 
 W_MemoryView.typedef = TypeDef(
-    "memoryview", None, None, "read-write",
+    "memoryview", None, None, "read-write", variable_sized=True,
     __doc__ = """\
 Create a new memoryview object which references the given object.
 """,
@@ -714,7 +711,6 @@
     __enter__   = interp2app(W_MemoryView.descr_enter),
     __exit__    = interp2app(W_MemoryView.descr_exit),
     __weakref__ = make_weakref_descr(W_MemoryView),
-    __getstate__= interp2app(W_MemoryView.descr___getstate__),
     cast        = interp2app(W_MemoryView.descr_cast),
     hex         = interp2app(W_MemoryView.descr_hex),
     tobytes     = interp2app(W_MemoryView.descr_tobytes),
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -24,9 +24,8 @@
     try:
         getstate = obj.__getstate__
     except AttributeError:
-        # TODO restrict pickling of variable sized objects
-        # tp_itemsize != 0, for now e.g. memoryview is handled
-        # by raising a TypeError in __getstate__
+        # and raises a TypeError if the condition holds true, this is done
+        # just before reduce_2 is called in pypy
         state = getattr(obj, "__dict__", None)
         names = slotnames(cls) # not checking for list
         if names is not None:
@@ -176,6 +175,7 @@
     w_proto = space.newint(proto)
     if proto >= 2:
         w_descr = space.lookup(w_obj, '__getnewargs_ex__')
+        hasargs = True
         if w_descr is not None:
             w_result = space.get_and_call_function(w_descr, w_obj)
             w_args, w_kwargs = space.fixedview(w_result, 2)
@@ -184,8 +184,17 @@
             if w_descr is not None:
                 w_args = space.get_and_call_function(w_descr, w_obj)
             else:
+                hasargs = False
                 w_args = space.newtuple([])
             w_kwargs = space.w_None
+        w_getstate = space.lookup(w_obj, '__get_state__')
+        if w_getstate is None:
+            required = not hasargs and \
+                       not space.isinstance_w(w_obj, space.w_list) and \
+                       not space.isinstance_w(w_obj, space.w_dict)
+            w_obj_type = space.type(w_obj)
+            if required and w_obj_type.layout.typedef.variable_sized:
+                raise oefmt(space.w_TypeError, "cannot pickle %N objects", w_obj_type)
         return reduce_2(space, w_obj, w_proto, w_args, w_kwargs)
     return reduce_1(space, w_obj, w_proto)
 

From pypy.commits at gmail.com  Tue Feb 21 09:41:51 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 21 Feb 2017 06:41:51 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-2: - some more fixes
Message-ID: <58ac51af.18532e0a.7645f.8f4a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-2
Changeset: r90271:38359f83e374
Date: 2017-02-21 15:41 +0100
http://bitbucket.org/pypy/pypy/changeset/38359f83e374/

Log:	- some more fixes
	- make the storage in Reader be unencoded

diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -13,7 +13,8 @@
 from rpython.rtyper.rclass import OBJECTPTR
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp import resumecode
-TagOverflow = resumecode.TagOverflow
+from rpython.jit.metainterp.resumecode import TagOverflow
+
 
 
 # Logic to encode the chain of frames and the state of the boxes at a
@@ -563,8 +564,8 @@
                     raise TagOverflow
                 #
                 rd_pendingfields[i].lldescr = lldescr
-                rd_pendingfields[i].num = num
-                rd_pendingfields[i].fieldnum = fieldnum
+                rd_pendingfields[i].num = rffi.r_short(num)
+                rd_pendingfields[i].fieldnum = rffi.r_short(fieldnum)
                 rd_pendingfields[i].itemindex = rffi.cast(rffi.INT, itemindex)
         self.storage.rd_pendingfields = rd_pendingfields
 
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -1,7 +1,7 @@
 """ Resume bytecode. It goes as following:
 
   # ----- resume section
-  [total size of resume section, unencoded]
+  [total size of resume section]
   [<length> <virtualizable object> <numb> <numb> <numb>]    if vinfo is not None
    -OR-
   [1 <ginfo object>]                                        if ginfo is not None
@@ -99,29 +99,29 @@
 
 class Writer(object):
     def __init__(self, size=0):
-        self.current = objectmodel.newlist_hint(3 * size)
-        self.grow(size)
-        self.items = 0
+        self.current = objectmodel.newlist_hint( size)
 
     def append_int(self, item):
-        self.items += 1
-        append_numbering(self.current, item)
+        """ append an item. return the position of the item """
+        if not integer_fits(item):
+            raise TagOverflow
+        self.current.append(item)
+        return len(self.current) - 1
 
     def create_numbering(self):
-        numb = lltype.malloc(NUMBERING, len(self.current))
-        for i, elt in enumerate(self.current):
+        l = []
+        for item in self.current:
+            append_numbering(l, item)
+        numb = lltype.malloc(NUMBERING, len(l))
+        for i, elt in enumerate(l):
             numb.code[i] = elt
         return numb
 
-    def grow(self, size):
-        pass
+    def patch_current_size(self, index):
+        self.patch(index, len(self.current))
 
-    def patch_current_size(self, index):
-        # mess :-(
-        assert self.current[index] == 0
-        l = []
-        append_numbering(l, self.items)
-        self.current = l + self.current[1:]
+    def patch(self, index, item):
+        self.current[index] = item
 
 
 def create_numbering(l):
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -113,19 +113,19 @@
 def test_tag():
     assert tag(3, 1) == 3<<2|1
     assert tag(-3, 2) == -3<<2|2
-    assert tag((1<<13)-1, 3) == ((1<<15)-1)|3
-    assert tag(-1<<13, 3) ==(-1<<15)|3
+    assert tag((1<<19)-1, 3) == ((1<<21)-1)|3
+    assert tag(-1<<19, 3) ==(-1<<21)|3
     py.test.raises(AssertionError, tag, 3, 5)
-    py.test.raises(TagOverflow, tag, 1<<13, 0)
-    py.test.raises(TagOverflow, tag, (1<<13)+1, 0)
-    py.test.raises(TagOverflow, tag, (-1<<13)-1, 0)
-    py.test.raises(TagOverflow, tag, (-1<<13)-5, 0)
+    py.test.raises(TagOverflow, tag, 1<<19, 0)
+    py.test.raises(TagOverflow, tag, (1<<19)+1, 0)
+    py.test.raises(TagOverflow, tag, (-1<<19)-1, 0)
+    py.test.raises(TagOverflow, tag, (-1<<19)-5, 0)
 
 def test_untag():
     assert untag(tag(3, 1)) == (3, 1)
     assert untag(tag(-3, 2)) == (-3, 2)
-    assert untag(tag((1<<13)-1, 3)) == ((1<<13)-1, 3)
-    assert untag(tag(-1<<13, 3)) == (-1<<13, 3)
+    assert untag(tag((1<<19)-1, 3)) == ((1<<19)-1, 3)
+    assert untag(tag(-1<<19, 3)) == (-1<<19, 3)
 
 def test_tagged_eq():
     assert tagged_eq(UNASSIGNED, UNASSIGNED)
@@ -776,12 +776,12 @@
     assert untag(tagged) == (44, TAGINT)
     tagged = memo.getconst(ConstInt(-3))
     assert untag(tagged) == (-3, TAGINT)
-    const = ConstInt(50000)
+    const = ConstInt(1<<20)
     tagged = memo.getconst(const)
     index, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert memo.consts[index - TAG_CONST_OFFSET] is const
-    tagged = memo.getconst(ConstInt(50000))
+    tagged = memo.getconst(ConstInt(1<<20))
     index2, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert index2 == index
@@ -1081,7 +1081,7 @@
     _next_section(reader, sys.maxint, 2**16, -65)
 
 def test_virtual_adder_memo_const_sharing():
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**22), ConstInt(-65)]
     storage, t = make_storage(b1s, b2s, b3s)
     metainterp_sd = FakeMetaInterpStaticData()
     memo = ResumeDataLoopMemo(metainterp_sd)
@@ -1091,7 +1091,7 @@
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**23), ConstInt(-65)]
     storage2, t = make_storage(b1s, b2s, b3s)
     i = t.get_iter()
     modifier2 = ResumeDataVirtualAdder(FakeOptimizer(i), storage2, storage2,
@@ -1497,14 +1497,12 @@
             self.got_array = got_array
         def setfield(self, struct, fieldnum, descr):
             assert lltype.typeOf(struct) is lltype.Signed
-            assert lltype.typeOf(fieldnum) is rffi.SHORT
-            fieldnum = rffi.cast(lltype.Signed, fieldnum)
+            assert lltype.typeOf(fieldnum) is lltype.Signed
             self.got.append((descr, struct, fieldnum))
         def setarrayitem(self, array, index, fieldnum, arraydescr):
             assert lltype.typeOf(array) is lltype.Signed
             assert lltype.typeOf(index) is lltype.Signed
-            assert lltype.typeOf(fieldnum) is rffi.SHORT
-            fieldnum = rffi.cast(lltype.Signed, fieldnum)
+            assert lltype.typeOf(fieldnum) is lltype.Signed
             self.got_array.append((arraydescr, array, index, fieldnum))
         def decode_ref(self, num):
             return rffi.cast(lltype.Signed, num) * 100
diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -20,8 +20,10 @@
     [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
 ]
 
+codelists = strategies.lists(strategies.integers(-2**21, 2**21-1), min_size=1)
+
 def hypothesis_and_examples(func):
-    func = given(strategies.lists(strategies.integers(-2**21, 2**21-1)))(func)
+    func = given(codelists)(func)
     for ex in examples:
         func = example(ex)(func)
     return func
@@ -55,7 +57,7 @@
         assert unpack_numbering(n) == l
 
 @hypothesis_and_examples
-def test_patch(l):
+def test_patch_current_size(l):
     for middle in range(len(l)):
         l1 = l[:middle]
         l2 = l[middle:]
@@ -70,4 +72,18 @@
         assert unpack_numbering(n)[1:] == l
         assert unpack_numbering(n)[0] == middle + 1
 
+ at hypothesis_and_examples
+def test_patch(l):
+    item = l[0]
+    l = l[1:]
+    for middle in range(len(l)):
+        output = l[:]
+        output[middle] = item
+        w = Writer(len(l))
+        for i, num in enumerate(l):
+            index = w.append_int(num)
+            assert index == i
+        w.patch(middle, item)
+        n = w.create_numbering()
+        assert unpack_numbering(n) == output
 

From pypy.commits at gmail.com  Tue Feb 21 10:36:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Feb 2017 07:36:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Implement PyUnicode_InternInPlace (port
 PyString_InternInPlace from default)
Message-ID: <58ac5e79.12a3190a.d5f9d.97ea@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90272:033d9de18f47
Date: 2017-02-21 15:35 +0000
http://bitbucket.org/pypy/pypy/changeset/033d9de18f47/

Log:	Implement PyUnicode_InternInPlace (port PyString_InternInPlace from
	default)

diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1858,28 +1858,6 @@
     there was an error."""
     raise NotImplementedError
 
- at cpython_api([PyObjectP], lltype.Void)
-def PyUnicode_InternInPlace(space, string):
-    """Intern the argument *string in place.  The argument must be the address of a
-    pointer variable pointing to a Python unicode string object.  If there is an
-    existing interned string that is the same as *string, it sets *string to
-    it (decrementing the reference count of the old string object and incrementing
-    the reference count of the interned string object), otherwise it leaves
-    *string alone and interns it (incrementing its reference count).
-    (Clarification: even though there is a lot of talk about reference counts, think
-    of this function as reference-count-neutral; you own the object after the call
-    if and only if you owned it before the call.)"""
-    raise NotImplementedError
-
-
- at cpython_api([rffi.CCHARP], PyObject)
-def PyUnicode_InternFromString(space, v):
-    """A combination of PyUnicode_FromString() and
-    PyUnicode_InternInPlace(), returning either a new unicode string object
-    that has been interned, or a new ("owned") reference to an earlier interned
-    string object with the same value."""
-    raise NotImplementedError
-
 
 @cpython_api([rffi.INT_real, CWCHARPP], rffi.INT_real, error=1)
 def Py_Main(space, argc, argv):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -78,6 +78,20 @@
         print(module.strlen(True))
         assert module.strlen(True) == 4
 
+    def test_intern_inplace(self):
+        module = self.import_extension('foo', [
+            ("test_intern_inplace", "METH_O",
+             '''
+                 PyObject *s = args;
+                 Py_INCREF(s);
+                 PyUnicode_InternInPlace(&s);
+                 return s;
+             '''
+             )])
+        # This does not test much, but at least the refcounts are checked.
+        assert module.test_intern_inplace('s') == 's'
+
+
     def test_unicode_buffer_init(self):
         module = self.import_extension('foo', [
             ("getunicode", "METH_NOARGS",
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -587,6 +587,23 @@
     w_str = space.newbytes(rffi.charp2str(s))
     return space.call_method(w_str, 'decode', space.newtext("utf-8"))
 
+ at cpython_api([PyObjectP], lltype.Void)
+def PyUnicode_InternInPlace(space, string):
+    """Intern the argument *string in place.  The argument must be the address
+    of a pointer variable pointing to a Python unicode string object.  If there
+    is an existing interned string that is the same as *string, it sets *string
+    to it (decrementing the reference count of the old string object and
+    incrementing the reference count of the interned string object), otherwise
+    it leaves *string alone and interns it (incrementing its reference count).
+    (Clarification: even though there is a lot of talk about reference counts,
+    think of this function as reference-count-neutral; you own the object after
+    the call if and only if you owned it before the call.)"""
+    w_str = from_ref(space, string[0])
+    w_str = space.new_interned_w_str(w_str)
+    Py_DecRef(space, string[0])
+    string[0] = make_ref(space, w_str)
+
+
 @cpython_api([CONST_STRING], PyObject)
 def PyUnicode_InternFromString(space, s):
     """A combination of PyUnicode_FromString() and

From pypy.commits at gmail.com  Tue Feb 21 10:54:30 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Feb 2017 07:54:30 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: raise on invalid isoltion levels (sqlite3)
Message-ID: <58ac62b6.85d3190a.4940c.9a61@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90273:ef5d1489274a
Date: 2017-02-21 16:44 +0100
http://bitbucket.org/pypy/pypy/changeset/ef5d1489274a/

Log:	raise on invalid isoltion levels (sqlite3)

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -190,6 +190,12 @@
                 self.cache[sql] = stat
         return stat
 
+BEGIN_STATMENTS = (
+    "BEGIN ",
+    "BEGIN DEFERRED",
+    "BEGIN IMMEDIATE",
+    "BEGIN EXCLUSIVE",
+)
 
 class Connection(object):
     __initialized = False
@@ -691,7 +697,13 @@
         if val is None:
             self.commit()
         else:
-            self.__begin_statement = str("BEGIN " + val).encode('utf-8')
+            if not isinstance(val, str):
+                raise TypeError("isolation level must be " \
+                        "a string or None, not %s" % type(val).__name__)
+            stmt = str("BEGIN " + val).upper()
+            if stmt not in BEGIN_STATMENTS:
+                raise ValueError("invalid value for isolation_level")
+            self.__begin_statement = stmt.encode('utf-8')
         self._isolation_level = val
     isolation_level = property(__get_isolation_level, __set_isolation_level)
 

From pypy.commits at gmail.com  Tue Feb 21 10:54:33 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Feb 2017 07:54:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: change exception message to pass cpython
 test
Message-ID: <58ac62b9.554c190a.c616c.9e15@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90274:620065d707da
Date: 2017-02-21 16:44 +0100
http://bitbucket.org/pypy/pypy/changeset/620065d707da/

Log:	change exception message to pass cpython test

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -951,7 +951,7 @@
         if isinstance(sql, unicode):
             sql = sql.encode('utf-8')
         elif not isinstance(sql, str):
-            raise ValueError("script argument must be unicode or string.")
+            raise ValueError("script argument must be unicode.")
         statement_star = _ffi.new('sqlite3_stmt **')
         next_char = _ffi.new('char **')
 

From pypy.commits at gmail.com  Tue Feb 21 10:54:35 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 21 Feb 2017 07:54:35 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: raise not supported error if version is
 too old
Message-ID: <58ac62bb.c59c190a.fbab7.9d69@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90275:6165ad0377f1
Date: 2017-02-21 16:52 +0100
http://bitbucket.org/pypy/pypy/changeset/6165ad0377f1/

Log:	raise not supported error if version is too old

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -245,6 +245,8 @@
         self.__collations = {}
         if check_same_thread:
             self.__thread_ident = threading.get_ident()
+        if not check_same_thread and _lib.sqlite3_libversion_number() < 3003001:
+            raise NotSupportedError("shared connections not available")
 
         self.Error = Error
         self.Warning = Warning
diff --git a/lib_pypy/_sqlite3_build.py b/lib_pypy/_sqlite3_build.py
--- a/lib_pypy/_sqlite3_build.py
+++ b/lib_pypy/_sqlite3_build.py
@@ -108,6 +108,7 @@
 static const long SQLITE_OPEN_CREATE;
 
 const char *sqlite3_libversion(void);
+int sqlite3_libversion_number(void);
 
 typedef ... sqlite3;
 typedef ... sqlite3_stmt;

From pypy.commits at gmail.com  Tue Feb 21 12:35:54 2017
From: pypy.commits at gmail.com (fijal)
Date: Tue, 21 Feb 2017 09:35:54 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: write down check_utf8
Message-ID: <58ac7a7a.1a4a2e0a.4e744.a716@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90276:07c779b7a698
Date: 2017-02-21 18:35 +0100
http://bitbucket.org/pypy/pypy/changeset/07c779b7a698/

Log:	write down check_utf8

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -52,16 +52,15 @@
         final=True, errorhandler=decode_error_handler(space))
     return result, length
 
-def decode_utf8(space, string):
+def check_utf8(space, string):
     # Surrogates are accepted and not treated specially at all.
     # If there happen to be two 3-bytes encoding a pair of surrogates,
     # you still get two surrogate unicode characters in the result.
     # These are the Python2 rules; Python3 differs.
-    result, consumed = runicode.str_decode_utf_8(
-        string, len(string), "strict",
-        final=True, errorhandler=decode_error_handler(space),
+    consumed, length = rutf8.str_check_utf8(
+        string, "strict", final=True, errorhandler=decode_error_handler(space),
         allow_surrogates=True)
-    return result
+    return length
 
 def encode_utf8(space, uni):
     # Note that this function never raises UnicodeEncodeError,
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -27,10 +27,10 @@
     "Return a Unicode string of one character with the given ordinal."
     # XXX this assumes unichr would be happy to return you surrogates
     try:
-        s = unichr_as_utf8(code)
+        s, lgt = unichr_as_utf8(code)
     except ValueError:
         raise oefmt(space.w_ValueError, "unichr() arg out of range")
-    return space.newunicode(s)
+    return space.newunicode(s, lgt)
 
 def len(space, w_obj):
     "len(object) -> integer\n\nReturn the number of items of a sequence or mapping."
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -403,8 +403,9 @@
 
 @unmarshaller(TYPE_UNICODE)
 def unmarshal_unicode(space, u, tc):
-    return space.newunicode(unicodehelper.decode_utf8(space, u.get_str()))
-
+    arg = u.get_str()
+    length = unicodehelper.check_utf8(space, arg)
+    return space.newunicode(arg, length)
 
 @marshaller(W_SetObject)
 def marshal_set(space, w_set, m):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -583,7 +583,7 @@
     except rutf8.AsciiCheckError:
         # raising UnicodeDecodeError is messy, "please crash for me"
         return utf8_from_encoded_object(space, w_bytes, "ascii", "strict")
-    return W_UnicodeObject(s)
+    return W_UnicodeObject(s, len(s))
 
 
 class UnicodeDocstrings:
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -1,26 +1,32 @@
 
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import runicode, jit
+from rpython.rlib.nonconst import NonConstant
+from rpython.tool.sourcetools import func_with_new_name
 
 def unichr_as_utf8(code):
     """ Encode code (numeric value) as utf8 encoded string
     """
     if code < 0:
         raise ValueError
+    lgt = 1
+    if code >= runicode.MAXUNICODE:
+        lgt = 2
     if code < 0x80:
         # Encode ASCII
-        return chr(code)
+        return chr(code), 1
     if code < 0x0800:
         # Encode Latin-1
-        return chr((0xc0 | (code >> 6))) + chr((0x80 | (code & 0x3f)))
+        return chr((0xc0 | (code >> 6))) + chr((0x80 | (code & 0x3f))), lgt
     if code < 0x10000:
         return (chr((0xe0 | (code >> 12))) +
                 chr((0x80 | ((code >> 6) & 0x3f))) +
-                chr((0x80 | (code & 0x3f))))
+                chr((0x80 | (code & 0x3f)))), lgt
     if code < 0x10ffff:
         return (chr((0xf0 | (code >> 18))) +
                 chr((0x80 | ((code >> 12) & 0x3f))) +
                 chr((0x80 | ((code >> 6) & 0x3f))) +
-                chr((0x80 | (code & 0x3f))))
+                chr((0x80 | (code & 0x3f)))), lgt
     raise ValueError
 
 class AsciiCheckError(Exception):
@@ -32,6 +38,151 @@
         if ord(s[i]) & 0x80:
             raise AsciiCheckError(i)
 
+def default_unicode_error_check(*args):
+    xxx
+
+# if you can't use the @elidable version, call str_check_utf8_impl()
+# directly
+ at jit.elidable
+def str_check_utf8(s, size, errors, final=False,
+                   errorhandler=None,
+                   allow_surrogates=runicode.allow_surrogate_by_default):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_check
+    # XXX unclear, fix
+    # NB. a bit messy because rtyper/rstr.py also calls the same
+    # function.  Make sure we annotate for the args it passes, too
+    if NonConstant(False):
+        s = NonConstant('?????')
+        size = NonConstant(12345)
+        errors = NonConstant('strict')
+        final = NonConstant(True)
+        WTF # goes here
+        errorhandler = ll_unicode_error_decode
+        allow_surrogates = NonConstant(True)
+    return str_check_utf8_elidable(s, size, errors, final, errorhandler,
+                                   allow_surrogates=allow_surrogates)
+
+def str_check_utf8_impl(s, size, errors, final, errorhandler,
+                        allow_surrogates):
+    if size == 0:
+        return 0, 0
+
+    pos = 0
+    lgt = 0
+    while pos < size:
+        ordch1 = ord(s[pos])
+        # fast path for ASCII
+        # XXX maybe use a while loop here
+        if ordch1 < 0x80:
+            lgt += 1
+            pos += 1
+            continue
+
+        n = ord(runicode._utf8_code_length[ordch1 - 0x80])
+        if pos + n > size:
+            if not final:
+                break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
+            charsleft = size - pos - 1 # either 0, 1, 2
+            # note: when we get the 'unexpected end of data' we need
+            # to care about the pos returned; it can be lower than size,
+            # in case we need to continue running this loop
+            if not charsleft:
+                # there's only the start byte and nothing else
+                errorhandler(errors, 'utf8', 'unexpected end of data',
+                             s, pos, pos+1)
+            ordch2 = ord(s[pos+1])
+            if n == 3:
+                # 3-bytes seq with only a continuation byte
+                if runicode._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+                    # second byte invalid, take the first and continue
+                    errorhandler(errors, 'utf8', 'invalid continuation byte',
+                                 s, pos, pos+1)
+                else:
+                    # second byte valid, but third byte missing
+                    errorhandler(errors, 'utf8', 'unexpected end of data',
+                                 s, pos, pos+2)
+            elif n == 4:
+                # 4-bytes seq with 1 or 2 continuation bytes
+                if runicode._invalid_byte_2_of_4(ordch1, ordch2):
+                    # second byte invalid, take the first and continue
+                    errorhandler(errors, 'utf8', 'invalid continuation byte',
+                                 s, pos, pos+1)
+                elif charsleft == 2 and runicode._invalid_byte_3_of_4(ord(s[pos+2])):
+                    # third byte invalid, take the first two and continue
+                    errorhandler(errors, 'utf8', 'invalid continuation byte',
+                                 s, pos, pos+2)
+                else:
+                    # there's only 1 or 2 valid cb, but the others are missing
+                    errorhandler(errors, 'utf8', 'unexpected end of data',
+                                 s, pos, pos+charsleft+1)
+            raise AssertionError("unreachable")
+
+        if n == 0:
+            errorhandler(errors, 'utf8', 'invalid start byte', s, pos, pos+1)
+        elif n == 1:
+            assert 0, "ascii should have gone through the fast path"
+
+        elif n == 2:
+            ordch2 = ord(s[pos+1])
+            if runicode._invalid_byte_2_of_2(ordch2):
+                errorhandler(errors, 'utf8', 'invalid continuation byte',
+                             s, pos, pos+2)
+                assert False, "unreachable"
+            # 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz
+            lgt += 1
+            pos += 2
+
+        elif n == 3:
+            ordch2 = ord(s[pos+1])
+            ordch3 = ord(s[pos+2])
+            if runicode._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+                errorhandler(errors, 'utf8', 'invalid continuation byte',
+                             s, pos, pos+1)
+                assert False, "unreachable"
+            elif runicode._invalid_byte_3_of_3(ordch3):
+                errorhandler(errors, 'utf8', 'invalid continuation byte',
+                             s, pos, pos+2)
+                assert False, "unreachable"
+            # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
+            lgt += 1
+            pos += 3
+
+        elif n == 4:
+            ordch2 = ord(s[pos+1])
+            ordch3 = ord(s[pos+2])
+            ordch4 = ord(s[pos+3])
+            if runicode._invalid_byte_2_of_4(ordch1, ordch2):
+                errorhandler(errors, 'utf8', 'invalid continuation byte',
+                             s, pos, pos+1)
+                assert False, "unreachable"
+            elif runicode._invalid_byte_3_of_4(ordch3):
+                errorhandler(errors, 'utf8', 'invalid continuation byte',
+                             s, pos, pos+2)
+                assert False, "unreachable"
+            elif runicode._invalid_byte_4_of_4(ordch4):
+                errorhandler(errors, 'utf8', 'invalid continuation byte',
+                             s, pos, pos+3)
+                assert False, "unreachable"
+            # 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz
+            c = (((ordch1 & 0x07) << 18) +      # 0b00000111
+                 ((ordch2 & 0x3F) << 12) +      # 0b00111111
+                 ((ordch3 & 0x3F) << 6) +       # 0b00111111
+                 (ordch4 & 0x3F))               # 0b00111111
+            if c <= runicode.MAXUNICODE:
+                lgt += 1
+            else:
+                # append the two surrogates:
+                lgt += 2
+            pos += 4
+
+    return pos, lgt
+str_check_utf8_elidable = jit.elidable(
+    func_with_new_name(str_check_utf8_impl, "str_check_utf8_elidable"))
+
+
 def str_decode_raw_utf8_escape(s, size, errors, final=False,
                                errorhandler=None):
     lgt = 0
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -5,7 +5,9 @@
 
 @given(strategies.integers(min_value=0, max_value=runicode.MAXUNICODE))
 def test_unichr_as_utf8(i):
-    assert rutf8.unichr_as_utf8(i) == runicode.UNICHR(i).encode('utf8')
+    u, lgt = rutf8.unichr_as_utf8(i)
+    r = runicode.UNICHR(i)
+    assert u == r.encode('utf8')
 
 @given(strategies.binary())
 def test_check_ascii(s):
@@ -22,6 +24,28 @@
     else:
         assert not raised
 
+def error_handler(errors, encoding, msg, char, start, end):
+    raise UnicodeDecodeError(encoding, char, start, end, msg)
+
+ at given(strategies.binary())
+def test_str_check_utf8(s):
+    try:
+        u = s.decode("utf8")
+        valid = True
+    except UnicodeDecodeError as e:
+        valid = False
+    try:
+        consumed, length = rutf8.str_check_utf8(s, len(s), None,
+            errorhandler=error_handler, final=True)
+    except UnicodeDecodeError as a:
+        assert not valid
+        assert a.start == e.start
+        # assert a.end == e.end, ideally
+    else:
+        assert valid
+        assert consumed == len(s)
+        assert length == len(u)
+
 @given(strategies.binary())
 def test_str_decode_raw_utf8_escape(uni):
     return # XXX fix details

From pypy.commits at gmail.com  Tue Feb 21 13:05:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 10:05:17 -0800 (PST)
Subject: [pypy-commit] pypy default: Rewrite str_decode_utf_8_impl() to
 produce the exact same error messages
Message-ID: <58ac815d.0163190a.c01d2.074c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90277:5e8ef7ce3887
Date: 2017-02-21 19:04 +0100
http://bitbucket.org/pypy/pypy/changeset/5e8ef7ce3887/

Log:	Rewrite str_decode_utf_8_impl() to produce the exact same error
	messages as CPython 2.7. (test by fijal)

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -133,23 +133,6 @@
 def _invalid_cont_byte(ordch):
     return ordch>>6 != 0x2    # 0b10
 
-_invalid_byte_2_of_2 = _invalid_cont_byte
-_invalid_byte_3_of_3 = _invalid_cont_byte
-_invalid_byte_3_of_4 = _invalid_cont_byte
-_invalid_byte_4_of_4 = _invalid_cont_byte
-
- at enforceargs(allow_surrogates=bool)
-def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
-    return (ordch2>>6 != 0x2 or    # 0b10
-            (ordch1 == 0xe0 and ordch2 < 0xa0)
-            # surrogates shouldn't be valid UTF-8!
-            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
-
-def _invalid_byte_2_of_4(ordch1, ordch2):
-    return (ordch2>>6 != 0x2 or    # 0b10
-            (ordch1 == 0xf0 and ordch2 < 0x90) or
-            (ordch1 == 0xf4 and ordch2 > 0x8f))
-
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates):
     if size == 0:
@@ -170,60 +153,20 @@
         if pos + n > size:
             if not final:
                 break
-            # argh, this obscure block of code is mostly a copy of
-            # what follows :-(
             charsleft = size - pos - 1 # either 0, 1, 2
             # note: when we get the 'unexpected end of data' we need
             # to care about the pos returned; it can be lower than size,
             # in case we need to continue running this loop
-            if not charsleft:
-                # there's only the start byte and nothing else
-                r, pos = errorhandler(errors, 'utf8',
-                                      'unexpected end of data',
-                                      s, pos, pos+1)
-                result.append(r)
-                continue
-            ordch2 = ord(s[pos+1])
-            if n == 3:
-                # 3-bytes seq with only a continuation byte
-                if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
-                    # second byte invalid, take the first and continue
-                    r, pos = errorhandler(errors, 'utf8',
-                                          'invalid continuation byte',
-                                          s, pos, pos+1)
-                    result.append(r)
-                    continue
-                else:
-                    # second byte valid, but third byte missing
-                    r, pos = errorhandler(errors, 'utf8',
-                                      'unexpected end of data',
-                                      s, pos, pos+2)
-                    result.append(r)
-                    continue
-            elif n == 4:
-                # 4-bytes seq with 1 or 2 continuation bytes
-                if _invalid_byte_2_of_4(ordch1, ordch2):
-                    # second byte invalid, take the first and continue
-                    r, pos = errorhandler(errors, 'utf8',
-                                          'invalid continuation byte',
-                                          s, pos, pos+1)
-                    result.append(r)
-                    continue
-                elif charsleft == 2 and _invalid_byte_3_of_4(ord(s[pos+2])):
-                    # third byte invalid, take the first two and continue
-                    r, pos = errorhandler(errors, 'utf8',
-                                          'invalid continuation byte',
-                                          s, pos, pos+2)
-                    result.append(r)
-                    continue
-                else:
-                    # there's only 1 or 2 valid cb, but the others are missing
-                    r, pos = errorhandler(errors, 'utf8',
-                                      'unexpected end of data',
-                                      s, pos, pos+charsleft+1)
-                    result.append(r)
-                    continue
-            raise AssertionError("unreachable")
+            endpos = pos + 1
+            if charsleft >= 1 and not _invalid_cont_byte(ord(s[pos+1])):
+                endpos = pos + 2
+                if charsleft >= 2 and not _invalid_cont_byte(ord(s[pos+2])):
+                    endpos = pos + 3
+            r, pos = errorhandler(errors, 'utf8',
+                                  'unexpected end of data',
+                                  s, pos, endpos)
+            result.append(r)
+            continue
 
         if n == 0:
             r, pos = errorhandler(errors, 'utf8',
@@ -236,7 +179,7 @@
 
         elif n == 2:
             ordch2 = ord(s[pos+1])
-            if _invalid_byte_2_of_2(ordch2):
+            if _invalid_cont_byte(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
@@ -250,41 +193,48 @@
         elif n == 3:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
-            if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+            if _invalid_cont_byte(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif _invalid_byte_3_of_3(ordch3):
+            elif _invalid_cont_byte(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
             # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
-            result.append(unichr(((ordch1 & 0x0F) << 12) +     # 0b00001111
-                                 ((ordch2 & 0x3F) << 6) +      # 0b00111111
-                                 (ordch3 & 0x3F)))             # 0b00111111
+            c = (((ordch1 & 0x0F) << 12) +     # 0b00001111
+                 ((ordch2 & 0x3F) << 6) +      # 0b00111111
+                 (ordch3 & 0x3F))              # 0b00111111
+            if c < 2048 or (0xd800 <= c <= 0xdfff and not allow_surrogates):
+                r, pos = errorhandler(errors, 'utf8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+2)
+                result.append(r)
+                continue
+            result.append(unichr(c))
             pos += 3
 
         elif n == 4:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
             ordch4 = ord(s[pos+3])
-            if _invalid_byte_2_of_4(ordch1, ordch2):
+            if _invalid_cont_byte(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif _invalid_byte_3_of_4(ordch3):
+            elif _invalid_cont_byte(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
-            elif _invalid_byte_4_of_4(ordch4):
+            elif _invalid_cont_byte(ordch4):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+3)
@@ -295,6 +245,12 @@
                  ((ordch2 & 0x3F) << 12) +      # 0b00111111
                  ((ordch3 & 0x3F) << 6) +       # 0b00111111
                  (ordch4 & 0x3F))               # 0b00111111
+            if c <= 65535 or c > 0x10ffff:
+                r, pos = errorhandler(errors, 'utf8',
+                                      'invalid continuation byte',
+                                      s, pos, pos+3)
+                result.append(r)
+                continue
             if c <= MAXUNICODE:
                 result.append(UNICHR(c))
             else:
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -700,6 +700,27 @@
             assert decoder(seq, len(seq), 'ignore', final=True
                            ) == (res, len(seq))
 
+    @settings(max_examples=10000)
+    @given(strategies.binary())
+    def test_str_check_utf8(self, s):
+        try:
+            u = s.decode("utf8")
+            valid = True
+        except UnicodeDecodeError as e:
+            valid = False
+        try:
+            result, length = runicode.str_decode_utf_8(s, len(s), None,
+                errorhandler=None, final=True, allow_surrogates=True)
+        except UnicodeDecodeError as a:
+            assert not valid
+            assert a.start == e.start
+            assert a.end == e.end
+            assert str(a) == str(e)
+        else:
+            assert valid
+            assert result == u
+            assert length == len(s)
+
 
 class TestEncoding(UnicodeTests):
     def test_all_ascii(self):

From pypy.commits at gmail.com  Tue Feb 21 14:17:32 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 21 Feb 2017 11:17:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix: a memoryview on a read-only mmap
 must be read-only
Message-ID: <58ac924c.cda2190a.684a0.a148@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90278:56963c746067
Date: 2017-02-21 19:16 +0000
http://bitbucket.org/pypy/pypy/changeset/56963c746067/

Log:	Fix: a memoryview on a read-only mmap must be read-only

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -20,8 +20,11 @@
 
     def buffer_w(self, space, flags):
         self.check_valid()
-        return MMapBuffer(self.space, self.mmap,
-                          bool(flags & space.BUF_WRITABLE))
+        readonly = (self.mmap.access == ACCESS_READ)
+        write_required = bool(flags & space.BUF_WRITABLE)
+        if write_required and readonly:
+            raise oefmt(space.w_BufferError, "Object is not writable.")
+        return MMapBuffer(self.space, self.mmap, readonly)
 
     def writebuf_w(self, space):
         self.check_writeable()
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -536,8 +536,9 @@
         f.close()
 
     def test_memoryview(self):
-        from mmap import mmap
-        f = open(self.tmpname + "y", "bw+")
+        from mmap import mmap, PROT_READ
+        filename = self.tmpname + "y"
+        f = open(filename, "bw+")
         f.write(b"foobar")
         f.flush()
         m = mmap(f.fileno(), 6)
@@ -549,10 +550,18 @@
         del b  # For CPython: "exported pointers exist"
         m.close()
         f.close()
+        with open(filename, "rb") as f:
+            m = mmap(f.fileno(), 6, prot=PROT_READ)
+            b = memoryview(m)
+            assert b.readonly is True
+            assert b[:] == b"foobar"
+            del b
+            m.close()
 
     def test_offset(self):
         from mmap import mmap, ALLOCATIONGRANULARITY
-        f = open(self.tmpname + "y", "wb+")
+        filename = self.tmpname + "y"
+        f = open(filename, "wb+")
         f.write(b"foobar" * ALLOCATIONGRANULARITY)
         f.flush()
         size = ALLOCATIONGRANULARITY

From pypy.commits at gmail.com  Tue Feb 21 15:06:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 12:06:20 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: add maciej's tentative dates
Message-ID: <58ac9dbc.59142e0a.af4fc.a4ea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5786:0f1ba73f9550
Date: 2017-02-21 21:06 +0100
http://bitbucket.org/pypy/extradoc/changeset/0f1ba73f9550/

Log:	add maciej's tentative dates

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -16,7 +16,7 @@
 Robert Zaremba       26?.02/01.03   Ermina
 Ronan Lamy           25.02/04.03    Ermina
 Antonio Cuni         27.02/03.03    Ermina
-Maciej Fijalkowski   maybe          Ermina
+Maciej Fijalkowski   27.02/29.02    (tentatively)
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Tue Feb 21 15:16:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 12:16:22 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: arrive Sunday
Message-ID: <58aca016.cfa8190a.ae52b.514c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5787:73ed31100fcd
Date: 2017-02-21 21:16 +0100
http://bitbucket.org/pypy/extradoc/changeset/73ed31100fcd/

Log:	arrive Sunday

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -16,7 +16,7 @@
 Robert Zaremba       26?.02/01.03   Ermina
 Ronan Lamy           25.02/04.03    Ermina
 Antonio Cuni         27.02/03.03    Ermina
-Maciej Fijalkowski   27.02/29.02    (tentatively)
+Maciej Fijalkowski   26.02/29.02    (tentatively)
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Tue Feb 21 15:26:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 12:26:12 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: confirmed
Message-ID: <58aca264.421a190a.d9f8.9db7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5788:dabae4d716ce
Date: 2017-02-21 21:25 +0100
http://bitbucket.org/pypy/extradoc/changeset/dabae4d716ce/

Log:	confirmed

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -16,7 +16,7 @@
 Robert Zaremba       26?.02/01.03   Ermina
 Ronan Lamy           25.02/04.03    Ermina
 Antonio Cuni         27.02/03.03    Ermina
-Maciej Fijalkowski   26.02/29.02    (tentatively)
+Maciej Fijalkowski   26.02/29.02    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Wed Feb 22 02:55:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Feb 2017 23:55:52 -0800 (PST)
Subject: [pypy-commit] pypy default: Backed out changeset 5e8ef7ce3887
Message-ID: <58ad4408.d4e1190a.f754c.05d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90279:94e61b03c50a
Date: 2017-02-21 19:14 +0100
http://bitbucket.org/pypy/pypy/changeset/94e61b03c50a/

Log:	Backed out changeset 5e8ef7ce3887

	Turns out that Python 2's utf-8 decoding is "wrong", and runicode.py
	implements a more correct version which follows Python 3 more
	closely. This is documented in the docstring of
	test_invalid_cb_for_3bytes_seq().

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -133,6 +133,23 @@
 def _invalid_cont_byte(ordch):
     return ordch>>6 != 0x2    # 0b10
 
+_invalid_byte_2_of_2 = _invalid_cont_byte
+_invalid_byte_3_of_3 = _invalid_cont_byte
+_invalid_byte_3_of_4 = _invalid_cont_byte
+_invalid_byte_4_of_4 = _invalid_cont_byte
+
+ at enforceargs(allow_surrogates=bool)
+def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xe0 and ordch2 < 0xa0)
+            # surrogates shouldn't be valid UTF-8!
+            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
+
+def _invalid_byte_2_of_4(ordch1, ordch2):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xf0 and ordch2 < 0x90) or
+            (ordch1 == 0xf4 and ordch2 > 0x8f))
+
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates):
     if size == 0:
@@ -153,20 +170,60 @@
         if pos + n > size:
             if not final:
                 break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
             charsleft = size - pos - 1 # either 0, 1, 2
             # note: when we get the 'unexpected end of data' we need
             # to care about the pos returned; it can be lower than size,
             # in case we need to continue running this loop
-            endpos = pos + 1
-            if charsleft >= 1 and not _invalid_cont_byte(ord(s[pos+1])):
-                endpos = pos + 2
-                if charsleft >= 2 and not _invalid_cont_byte(ord(s[pos+2])):
-                    endpos = pos + 3
-            r, pos = errorhandler(errors, 'utf8',
-                                  'unexpected end of data',
-                                  s, pos, endpos)
-            result.append(r)
-            continue
+            if not charsleft:
+                # there's only the start byte and nothing else
+                r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+1)
+                result.append(r)
+                continue
+            ordch2 = ord(s[pos+1])
+            if n == 3:
+                # 3-bytes seq with only a continuation byte
+                if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+                    # second byte invalid, take the first and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                else:
+                    # second byte valid, but third byte missing
+                    r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+2)
+                    result.append(r)
+                    continue
+            elif n == 4:
+                # 4-bytes seq with 1 or 2 continuation bytes
+                if _invalid_byte_2_of_4(ordch1, ordch2):
+                    # second byte invalid, take the first and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                elif charsleft == 2 and _invalid_byte_3_of_4(ord(s[pos+2])):
+                    # third byte invalid, take the first two and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+2)
+                    result.append(r)
+                    continue
+                else:
+                    # there's only 1 or 2 valid cb, but the others are missing
+                    r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+charsleft+1)
+                    result.append(r)
+                    continue
+            raise AssertionError("unreachable")
 
         if n == 0:
             r, pos = errorhandler(errors, 'utf8',
@@ -179,7 +236,7 @@
 
         elif n == 2:
             ordch2 = ord(s[pos+1])
-            if _invalid_cont_byte(ordch2):
+            if _invalid_byte_2_of_2(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
@@ -193,48 +250,41 @@
         elif n == 3:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
-            if _invalid_cont_byte(ordch2):
+            if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif _invalid_cont_byte(ordch3):
+            elif _invalid_byte_3_of_3(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
             # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
-            c = (((ordch1 & 0x0F) << 12) +     # 0b00001111
-                 ((ordch2 & 0x3F) << 6) +      # 0b00111111
-                 (ordch3 & 0x3F))              # 0b00111111
-            if c < 2048 or (0xd800 <= c <= 0xdfff and not allow_surrogates):
-                r, pos = errorhandler(errors, 'utf8',
-                                      'invalid continuation byte',
-                                      s, pos, pos+2)
-                result.append(r)
-                continue
-            result.append(unichr(c))
+            result.append(unichr(((ordch1 & 0x0F) << 12) +     # 0b00001111
+                                 ((ordch2 & 0x3F) << 6) +      # 0b00111111
+                                 (ordch3 & 0x3F)))             # 0b00111111
             pos += 3
 
         elif n == 4:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
             ordch4 = ord(s[pos+3])
-            if _invalid_cont_byte(ordch2):
+            if _invalid_byte_2_of_4(ordch1, ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif _invalid_cont_byte(ordch3):
+            elif _invalid_byte_3_of_4(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
-            elif _invalid_cont_byte(ordch4):
+            elif _invalid_byte_4_of_4(ordch4):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+3)
@@ -245,12 +295,6 @@
                  ((ordch2 & 0x3F) << 12) +      # 0b00111111
                  ((ordch3 & 0x3F) << 6) +       # 0b00111111
                  (ordch4 & 0x3F))               # 0b00111111
-            if c <= 65535 or c > 0x10ffff:
-                r, pos = errorhandler(errors, 'utf8',
-                                      'invalid continuation byte',
-                                      s, pos, pos+3)
-                result.append(r)
-                continue
             if c <= MAXUNICODE:
                 result.append(UNICHR(c))
             else:
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -700,27 +700,6 @@
             assert decoder(seq, len(seq), 'ignore', final=True
                            ) == (res, len(seq))
 
-    @settings(max_examples=10000)
-    @given(strategies.binary())
-    def test_str_check_utf8(self, s):
-        try:
-            u = s.decode("utf8")
-            valid = True
-        except UnicodeDecodeError as e:
-            valid = False
-        try:
-            result, length = runicode.str_decode_utf_8(s, len(s), None,
-                errorhandler=None, final=True, allow_surrogates=True)
-        except UnicodeDecodeError as a:
-            assert not valid
-            assert a.start == e.start
-            assert a.end == e.end
-            assert str(a) == str(e)
-        else:
-            assert valid
-            assert result == u
-            assert length == len(s)
-
 
 class TestEncoding(UnicodeTests):
     def test_all_ascii(self):

From pypy.commits at gmail.com  Wed Feb 22 03:08:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 00:08:25 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in fniephaus/pypy/space-wrap
 (pull request #518)
Message-ID: <58ad46f9.c35c2e0a.28188.1e3a@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r90281:1c6f372c0035
Date: 2017-02-22 08:07 +0000
http://bitbucket.org/pypy/pypy/changeset/1c6f372c0035/

Log:	Merged in fniephaus/pypy/space-wrap (pull request #518)

	Avoid space.wrap when possible

	Approved-by: Armin Rigo

diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -18,7 +18,8 @@
 
 def compilecode(space, source, filename, cmd='exec'):
     w_code = space.builtin.call(
-        'compile', space.wrap(source), space.wrap(filename), space.wrap(cmd), space.newint(0), space.newint(0))
+        'compile', space.newtext(source), space.newtext(filename),
+        space.newtext(cmd), space.newint(0), space.newint(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
 
@@ -85,10 +86,11 @@
         argv.extend(args)
     space.setitem(space.sys.w_dict, space.newtext('argv'), space.wrap(argv))
     w_import = space.builtin.get('__import__')
-    runpy = space.call_function(w_import, space.wrap('runpy'))
-    w_run_module = space.getitem(runpy.w_dict, space.wrap('run_module'))
-    return space.call_function(w_run_module, space.wrap(module_name), space.w_None,
-                               space.wrap('__main__'), space.w_True)
+    runpy = space.call_function(w_import, space.newtext('runpy'))
+    w_run_module = space.getitem(runpy.w_dict, space.newtext('run_module'))
+    return space.call_function(w_run_module, space.newtext(module_name),
+                               space.w_None, space.newtext('__main__'),
+                               space.w_True)
 
 
 def run_toplevel(space, f, verbose=False):

From pypy.commits at gmail.com  Wed Feb 22 03:08:35 2017
From: pypy.commits at gmail.com (fniephaus)
Date: Wed, 22 Feb 2017 00:08:35 -0800 (PST)
Subject: [pypy-commit] pypy space-wrap: Avoid space.wrap when possible
Message-ID: <58ad4703.42072e0a.284a8.4910@mx.google.com>

Author: Fabio Niephaus <code at fniephaus.com>
Branch: space-wrap
Changeset: r90280:4dbfbf63a6fb
Date: 2017-02-22 08:06 +0100
http://bitbucket.org/pypy/pypy/changeset/4dbfbf63a6fb/

Log:	Avoid space.wrap when possible

diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -18,7 +18,8 @@
 
 def compilecode(space, source, filename, cmd='exec'):
     w_code = space.builtin.call(
-        'compile', space.wrap(source), space.wrap(filename), space.wrap(cmd), space.newint(0), space.newint(0))
+        'compile', space.newtext(source), space.newtext(filename),
+        space.newtext(cmd), space.newint(0), space.newint(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
 
@@ -85,10 +86,11 @@
         argv.extend(args)
     space.setitem(space.sys.w_dict, space.newtext('argv'), space.wrap(argv))
     w_import = space.builtin.get('__import__')
-    runpy = space.call_function(w_import, space.wrap('runpy'))
-    w_run_module = space.getitem(runpy.w_dict, space.wrap('run_module'))
-    return space.call_function(w_run_module, space.wrap(module_name), space.w_None,
-                               space.wrap('__main__'), space.w_True)
+    runpy = space.call_function(w_import, space.newtext('runpy'))
+    w_run_module = space.getitem(runpy.w_dict, space.newtext('run_module'))
+    return space.call_function(w_run_module, space.newtext(module_name),
+                               space.w_None, space.newtext('__main__'),
+                               space.w_True)
 
 
 def run_toplevel(space, f, verbose=False):

From pypy.commits at gmail.com  Wed Feb 22 03:32:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 00:32:38 -0800 (PST)
Subject: [pypy-commit] pypy default: Add comment
Message-ID: <58ad4ca6.55582e0a.d9c4c.b66e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90282:67b56d3a90e0
Date: 2017-02-22 09:22 +0100
http://bitbucket.org/pypy/pypy/changeset/67b56d3a90e0/

Log:	Add comment

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -150,6 +150,9 @@
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
+# NOTE: this is a slightly fixed algorithm when compared with
+# CPython2's.  It is closer to CPython3's.  See comments in
+# test_invalid_cb_for_3bytes_seq().
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates):
     if size == 0:

From pypy.commits at gmail.com  Wed Feb 22 03:32:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 00:32:41 -0800 (PST)
Subject: [pypy-commit] pypy default: Get rid of the RUnicodeEncodeError
 exception class, used only at one
Message-ID: <58ad4ca9.1a0e2e0a.4eca.b65f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90283:c0dbd36ded3d
Date: 2017-02-22 09:31 +0100
http://bitbucket.org/pypy/pypy/changeset/c0dbd36ded3d/

Log:	Get rid of the RUnicodeEncodeError exception class, used only at one
	place. It is turned into a space.w_UnicodeEncodeError immediately
	afterwards anyway.

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -16,17 +16,18 @@
                                              space.newtext(msg)]))
     return raise_unicode_exception_decode
 
-class RUnicodeEncodeError(Exception):
-    def __init__(self, encoding, object, start, end, reason):
-        self.encoding = encoding
-        self.object = object
-        self.start = start
-        self.end = end
-        self.reason = reason
-
-def raise_unicode_exception_encode(errors, encoding, msg, u,
-                                   startingpos, endingpos):
-    raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
+ at specialize.memo()
+def encode_error_handler(space):
+    # Fast version of the "strict" errors handler.
+    def raise_unicode_exception_encode(errors, encoding, msg, u,
+                                       startingpos, endingpos):
+        raise OperationError(space.w_UnicodeEncodeError,
+                             space.newtuple([space.newtext(encoding),
+                                             space.newunicode(u),
+                                             space.newint(startingpos),
+                                             space.newint(endingpos),
+                                             space.newtext(msg)]))
+    return raise_unicode_exception_encode
 
 # ____________________________________________________________
 
@@ -68,5 +69,5 @@
     # it stands for.  These are the Python2 rules; Python3 differs.
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
-        errorhandler=raise_unicode_exception_encode,
+        errorhandler=None,
         allow_surrogates=True)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -462,26 +462,17 @@
         w_encoder = space.sys.get_w_default_encoder()
     else:
         if errors is None or errors == 'strict':
-            try:
-                if encoding == 'ascii':
-                    u = space.unicode_w(w_object)
-                    eh = unicodehelper.raise_unicode_exception_encode
-                    return space.newbytes(unicode_encode_ascii(
-                            u, len(u), None, errorhandler=eh))
-                if encoding == 'utf-8':
-                    u = space.unicode_w(w_object)
-                    eh = unicodehelper.raise_unicode_exception_encode
-                    return space.newbytes(unicode_encode_utf_8(
-                            u, len(u), None, errorhandler=eh,
-                            allow_surrogates=True))
-            except unicodehelper.RUnicodeEncodeError as ue:
-                raise OperationError(space.w_UnicodeEncodeError,
-                                     space.newtuple([
-                    space.newtext(ue.encoding),
-                    space.newunicode(ue.object),
-                    space.newint(ue.start),
-                    space.newint(ue.end),
-                    space.newtext(ue.reason)]))
+            if encoding == 'ascii':
+                u = space.unicode_w(w_object)
+                eh = unicodehelper.encode_error_handler(space)
+                return space.newbytes(unicode_encode_ascii(
+                        u, len(u), None, errorhandler=eh))
+            if encoding == 'utf-8':
+                u = space.unicode_w(w_object)
+                eh = unicodehelper.encode_error_handler(space)
+                return space.newbytes(unicode_encode_utf_8(
+                        u, len(u), None, errorhandler=eh,
+                        allow_surrogates=True))
         from pypy.module._codecs.interp_codecs import lookup_codec
         w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
     if errors is None:

From pypy.commits at gmail.com  Wed Feb 22 03:44:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 00:44:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58ad4f86.4395190a.2afe7.0d27@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90284:12c942ae2993
Date: 2017-02-22 09:44 +0100
http://bitbucket.org/pypy/pypy/changeset/12c942ae2993/

Log:	hg merge default

diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -18,8 +18,8 @@
 
 def compilecode(space, source, filename, cmd='exec'):
     w_code = space.builtin.call(
-        'compile', space.newbytes(source), space.wrap_fsdecoded(filename),
-        space.wrap(cmd), space.newint(0), space.newint(0))
+        'compile', space.newbytes(source), space.newfilename(filename),
+        space.newtext(cmd), space.newint(0), space.newint(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
 
@@ -87,10 +87,11 @@
         argv.extend(args)
     space.setitem(space.sys.w_dict, space.newtext('argv'), space.wrap(argv))
     w_import = space.builtin.get('__import__')
-    runpy = space.call_function(w_import, space.wrap('runpy'))
-    w_run_module = space.getitem(runpy.w_dict, space.wrap('run_module'))
-    return space.call_function(w_run_module, space.wrap(module_name), space.w_None,
-                               space.wrap('__main__'), space.w_True)
+    runpy = space.call_function(w_import, space.newtext('runpy'))
+    w_run_module = space.getitem(runpy.w_dict, space.newtext('run_module'))
+    return space.call_function(w_run_module, space.newtext(module_name),
+                               space.w_None, space.newtext('__main__'),
+                               space.w_True)
 
 
 def run_toplevel(space, f, verbose=False):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -37,23 +37,6 @@
                                              space.newtext(msg)]))
     return raise_unicode_exception_encode
 
-class RUnicodeEncodeError(Exception):
-    def __init__(self, encoding, object, start, end, reason):
-        assert isinstance(object, unicode)
-        self.encoding = encoding
-        self.object = object
-        self.start = start
-        self.end = end
-        self.reason = reason
-
- at specialize.memo()
-def rpy_encode_error_handler():
-    # A RPython version of the "strict" error handler.
-    def raise_unicode_exception_encode(errors, encoding, msg, u,
-                                       startingpos, endingpos):
-        raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
-    return raise_unicode_exception_encode
-
 # ____________________________________________________________
 
 def fsdecode(space, w_string):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -96,12 +96,9 @@
             if ignore_sg:
                 identifier = unicode_encode_utf8sp(u, len(u))
             else:
-                eh = unicodehelper.rpy_encode_error_handler()
-                try:
-                    identifier = unicode_encode_utf_8(u, len(u), None,
-                                                      errorhandler=eh)
-                except unicodehelper.RUnicodeEncodeError as ue:
-                    raise wrap_encode_error(space, ue)
+                eh = unicodehelper.encode_error_handler(space)
+                identifier = unicode_encode_utf_8(u, len(u), None,
+                                                  errorhandler=eh)
         return identifier
 
     def text_w(self, space):
@@ -576,19 +573,16 @@
 
 def encode_object(space, w_object, encoding, errors):
     if errors is None or errors == 'strict':
-        try:
-            if encoding is None or encoding == 'utf-8':
-                u = space.unicode_w(w_object)
-                eh = unicodehelper.encode_error_handler(space)
-                return space.newbytes(unicode_encode_utf_8(
-                        u, len(u), errors, errorhandler=eh))
-            elif encoding == 'ascii':
-                u = space.unicode_w(w_object)
-                eh = unicodehelper.encode_error_handler(space)
-                return space.newbytes(unicode_encode_ascii(
-                        u, len(u), errors, errorhandler=eh))
-        except unicodehelper.RUnicodeEncodeError as ue:
-            raise wrap_encode_error(space, ue)
+        if encoding is None or encoding == 'utf-8':
+            u = space.unicode_w(w_object)
+            eh = unicodehelper.encode_error_handler(space)
+            return space.newbytes(unicode_encode_utf_8(
+                    u, len(u), errors, errorhandler=eh))
+        elif encoding == 'ascii':
+            u = space.unicode_w(w_object)
+            eh = unicodehelper.encode_error_handler(space)
+            return space.newbytes(unicode_encode_ascii(
+                    u, len(u), errors, errorhandler=eh))
 
     from pypy.module._codecs.interp_codecs import encode_text
     w_retval = encode_text(space, w_object, encoding, errors)
@@ -599,16 +593,6 @@
     return w_retval
 
 
-def wrap_encode_error(space, ue):
-    raise OperationError(space.w_UnicodeEncodeError,
-                         space.newtuple([
-        space.newtext(ue.encoding),
-        space.newunicode(ue.object),
-        space.newint(ue.start),
-        space.newint(ue.end),
-        space.newtext(ue.reason)]))
-
-
 def decode_object(space, w_obj, encoding, errors):
     if encoding is None:
         encoding = getdefaultencoding(space)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -150,6 +150,9 @@
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
+# NOTE: this is a slightly fixed algorithm when compared with
+# CPython2's.  It is closer to CPython3's.  See comments in
+# test_invalid_cb_for_3bytes_seq().
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates):
     if size == 0:

From pypy.commits at gmail.com  Wed Feb 22 03:49:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 00:49:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: port of 5c0c182
Message-ID: <58ad50a7.02a5190a.90486.0c6e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90285:bbfa74ae6b0d
Date: 2017-02-22 09:48 +0100
http://bitbucket.org/pypy/pypy/changeset/bbfa74ae6b0d/

Log:	port of 5c0c182

diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -578,6 +578,9 @@
             import _osx_support
             _osx_support.customize_config_vars(_CONFIG_VARS)
 
+        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+                                                 'include')
+
     if args:
         vals = []
         for name in args:

From pypy.commits at gmail.com  Wed Feb 22 06:10:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 03:10:56 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove space.str_w() and space.str0_w()
 in RPython code.
Message-ID: <58ad71c0.51072e0a.d002.5729@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90286:a87a8bb34ecd
Date: 2017-02-22 12:09 +0100
http://bitbucket.org/pypy/pypy/changeset/a87a8bb34ecd/

Log:	Remove space.str_w() and space.str0_w() in RPython code. Remove
	``str`` as an unwrap_spec argument.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1617,12 +1617,12 @@
         unwrapped RPython bytestring."""
         return w_obj.str_w(self)
 
-    #@not_rpython    BACKCOMPAT: should be replaced with bytes_w or text_w
+    @not_rpython    # tests only; should be replaced with bytes_w or text_w
     def str_w(self, w_obj):
         """For tests only."""
         return self.bytes_w(w_obj)
 
-    #@not_rpython    BACKCOMPAT
+    @not_rpython    # tests only; should be replaced with bytes0_w or text0_w
     def str0_w(self, w_obj):
         return self.bytes0_w(w_obj)
 
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -148,9 +148,6 @@
     def visit_str_or_None(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
-    def visit_str0(self, el, app_sig):
-        self.checked_space_method(el, app_sig)
-
     def visit_bytes(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -292,9 +289,6 @@
     def visit_str_or_None(self, typ):
         self.run_args.append("space.str_or_None_w(%s)" % (self.scopenext(),))
 
-    def visit_str0(self, typ):
-        self.run_args.append("space.str0_w(%s)" % (self.scopenext(),))
-
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
 
@@ -457,9 +451,6 @@
     def visit_str_or_None(self, typ):
         self.unwrap.append("space.str_or_None_w(%s)" % (self.nextarg(),))
 
-    def visit_str0(self, typ):
-        self.unwrap.append("space.str0_w(%s)" % (self.nextarg(),))
-
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
 
@@ -606,6 +597,8 @@
                              "the name of an argument of the following "
                              "function" % (name,))
 
+    assert str not in unwrap_spec   # use 'text' or 'bytes' instead of str
+
     return unwrap_spec
 
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -48,7 +48,7 @@
             space.call_method(self.w_dict, 'update', self.w_initialdict)
 
         for w_submodule in self.submodules_w:
-            name = space.str0_w(w_submodule.w_name)
+            name = space.text0_w(w_submodule.w_name)
             space.setitem(self.w_dict, space.newtext(name.split(".")[-1]), w_submodule)
             space.getbuiltinmodule(name)
 
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -42,7 +42,7 @@
 
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
-        modulename = self.space.str0_w(self.w_name)
+        modulename = self.space.text0_w(self.w_name)
         self.space.builtin_modules[modulename] = self
 
     def setup_after_space_initialization(self):
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -377,7 +377,7 @@
             return space.wrap(s0+s1)
         app_g3_ss = gateway.interp2app_temp(g3_ss,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      str, 'str_or_None'])
+                                                      'text', 'str_or_None'])
         w_app_g3_ss = space.wrap(app_g3_ss)
         assert self.space.eq_w(
             space.call(w_app_g3_ss,
@@ -512,7 +512,7 @@
 
         app_g3_s = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      str])
+                                                      'text'])
         w_app_g3_s = space.wrap(app_g3_s)
         assert space.eq_w(space.call_function(w_app_g3_s,w("foo")),w("foo"))
         raises(gateway.OperationError,space.call_function,w_app_g3_s,w(None))
diff --git a/pypy/module/__pypy__/interp_os.py b/pypy/module/__pypy__/interp_os.py
--- a/pypy/module/__pypy__/interp_os.py
+++ b/pypy/module/__pypy__/interp_os.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 
 
- at unwrap_spec(name='str0')
+ at unwrap_spec(name='text0')
 def real_getenv(space, name):
     """Get an OS environment value skipping Python cache"""
     return space.newtext_or_none(os.environ.get(name))
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -6,7 +6,7 @@
 import pypy.module.cpyext.api     # side-effect of pre-importing it
 
 
- at unwrap_spec(cdef=str, module_name=str, source=str, packed=int)
+ at unwrap_spec(cdef='text', module_name='text', source='text', packed=int)
 def prepare(space, cdef, module_name, source, w_includes=None,
             w_extra_source=None, w_min_version=None, packed=False):
     try:
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -114,7 +114,7 @@
 # __________________________________________________________
 # functions for the "win32" namespace
 
- at unwrap_spec(name=str, openmode=r_uint, pipemode=r_uint, maxinstances=r_uint,
+ at unwrap_spec(name='text', openmode=r_uint, pipemode=r_uint, maxinstances=r_uint,
              outputsize=r_uint, inputsize=r_uint, timeout=r_uint)
 def CreateNamedPipe(space, name, openmode, pipemode, maxinstances,
                     outputsize, inputsize, timeout, w_security):
@@ -161,13 +161,13 @@
         lltype.free(state, flavor='raw')
         lltype.free(statep, flavor='raw')
 
- at unwrap_spec(name=str, timeout=r_uint)
+ at unwrap_spec(name='text', timeout=r_uint)
 def WaitNamedPipe(space, name, timeout):
     # Careful: zero means "default value specified by CreateNamedPipe()"
     if not _WaitNamedPipe(name, timeout):
         raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
- at unwrap_spec(filename=str, access=r_uint, share=r_uint,
+ at unwrap_spec(filename='fsencode', access=r_uint, share=r_uint,
              disposition=r_uint, flags=r_uint)
 def CreateFile(space, filename, access, share, w_security,
                disposition, flags, w_templatefile):
diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py
--- a/pypy/module/_rawffi/alt/test/test_struct.py
+++ b/pypy/module/_rawffi/alt/test/test_struct.py
@@ -43,7 +43,7 @@
     def setup_class(cls):
         BaseAppTestFFI.setup_class.im_func(cls)
 
-        @unwrap_spec(addr=int, typename=str, length=int)
+        @unwrap_spec(addr=int, typename='text', length=int)
         def read_raw_mem(space, addr, typename, length):
             import ctypes
             addr = ctypes.cast(addr, ctypes.c_void_p)
diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -99,7 +99,7 @@
                     usage.c_rgpszUsageIdentifier[i]))
             return space.newset(result_w)
 
- at unwrap_spec(store_name=str)
+ at unwrap_spec(store_name='text')
 def enum_certificates_w(space, store_name):
     """enum_certificates(store_name) -> []
 
@@ -142,7 +142,7 @@
 
     return space.newlist(result_w)
 
- at unwrap_spec(store_name=str)
+ at unwrap_spec(store_name='text')
 def enum_crls_w(space, store_name):
     """enum_crls(store_name) -> []
 
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -328,7 +328,7 @@
         raise oefmt(space.w_ValueError, "cannot open in read-write mode")
     if basemode == "a":
         raise oefmt(space.w_ValueError, "cannot append to bz2 file")
-    stream = open_path_helper(space.str0_w(w_path), os_flags, False)
+    stream = open_path_helper(space.fsencode_w(w_path), os_flags, False)
     if reading:
         bz2stream = ReadBZ2Filter(space, stream, buffering)
         buffering = 0     # by construction, the ReadBZ2Filter acts like
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -28,7 +28,7 @@
         data = DATA[:100]
         f.write(data, 'wb')
 
-    @unwrap_spec(data=str)
+    @unwrap_spec(data='bytes')
     def decompress(space, data):
         import popen2
         import bz2
diff --git a/pypy/module/cppyy/test/test_crossing.py b/pypy/module/cppyy/test/test_crossing.py
--- a/pypy/module/cppyy/test/test_crossing.py
+++ b/pypy/module/cppyy/test/test_crossing.py
@@ -78,7 +78,7 @@
             import ctypes, cppyy""")    # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
-        @unwrap_spec(name=str, init=str, body=str)
+        @unwrap_spec(name='text', init='text', body='text')
         def create_cdll(space, name, init, body):
             # the following is loosely from test_cpyext.py import_module; it
             # is copied here to be able to tweak the call to
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -287,7 +287,7 @@
         if self.runappdirect:
             return
 
-        @unwrap_spec(name=str)
+        @unwrap_spec(name='text')
         def compile_module(space, name,
                            w_source_files=None,
                            w_source_strings=None):
@@ -313,7 +313,7 @@
 
             return space.wrap(pydname)
 
-        @unwrap_spec(name=str, init='str_or_None', body=str,
+        @unwrap_spec(name='text', init='str_or_None', body='text',
                      filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
         def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
@@ -325,12 +325,12 @@
             return w_result
 
 
-        @unwrap_spec(mod=str, name=str)
+        @unwrap_spec(mod='text', name='text')
         def load_module(space, mod, name):
             return self.sys_info.load_module(mod, name)
 
-        @unwrap_spec(modname=str, prologue=str,
-                             more_init=str, PY_SSIZE_T_CLEAN=bool)
+        @unwrap_spec(modname='text', prologue='text',
+                             more_init='text', PY_SSIZE_T_CLEAN=bool)
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -79,7 +79,7 @@
 
 # ____________________________________________________________
 
- at unwrap_spec(filename='str0')
+ at unwrap_spec(filename='fsencode')
 def dump_heap_stats(space, filename):
     tb = rgc._heap_stats()
     if not tb:
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -491,7 +491,7 @@
     def __init__(self, space):
         pass
 
-    @unwrap_spec(path='str0')
+    @unwrap_spec(path='fsencode')
     def descr_init(self, space, path):
         if not path:
             raise oefmt(space.w_ImportError, "empty pathname")
@@ -570,7 +570,7 @@
                 if w_loader:
                     return FindInfo.fromLoader(w_loader)
 
-            path = space.str0_w(w_pathitem)
+            path = space.fsencode_w(w_pathitem)
             filepart = os.path.join(path, partname)
             log_pyverbose(space, 2, "# trying %s\n" % (filepart,))
             if os.path.isdir(filepart) and case_ok(filepart):
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -78,7 +78,7 @@
 def load_module(space, w_name, w_file, w_filename, w_info):
     w_suffix, w_filemode, w_modtype = space.unpackiterable(w_info, 3)
 
-    filename = space.str0_w(w_filename)
+    filename = space.fsencode_w(w_filename)
     filemode = space.text_w(w_filemode)
     if space.is_w(w_file, space.w_None):
         stream = None
@@ -95,7 +95,7 @@
         space, w_name, find_info, reuse=True)
 
 def load_source(space, w_modulename, w_filename, w_file=None):
-    filename = space.str0_w(w_filename)
+    filename = space.fsencode_w(w_filename)
 
     stream = get_file(space, w_file, filename, 'U')
 
@@ -109,7 +109,7 @@
         stream.close()
     return w_mod
 
- at unwrap_spec(filename='str0', check_afterwards=int)
+ at unwrap_spec(filename='fsencode', check_afterwards=int)
 def _run_compiled_module(space, w_modulename, filename, w_file, w_module,
                          check_afterwards=False):
     # the function 'imp._run_compiled_module' is a pypy-only extension
@@ -125,14 +125,14 @@
         stream.close()
     return w_mod
 
- at unwrap_spec(filename='str0')
+ at unwrap_spec(filename='fsencode')
 def load_compiled(space, w_modulename, filename, w_file=None):
     w_mod = Module(space, w_modulename)
     importing._prepare_module(space, w_mod, filename, None)
     return _run_compiled_module(space, w_modulename, filename, w_file, w_mod,
                                 check_afterwards=True)
 
- at unwrap_spec(filename='text')
+ at unwrap_spec(filename='fsencode')
 def load_dynamic(space, w_modulename, filename, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -147,17 +147,17 @@
         return None
 
 
- at unwrap_spec(executable='str0')
+ at unwrap_spec(executable='fsencode')
 def pypy_find_executable(space, executable):
     return space.newtext(find_executable(executable))
 
 
- at unwrap_spec(filename='str0')
+ at unwrap_spec(filename='fsencode')
 def pypy_resolvedirof(space, filename):
     return space.newtext(resolvedirof(filename))
 
 
- at unwrap_spec(executable='str0')
+ at unwrap_spec(executable='fsencode')
 def pypy_find_stdlib(space, executable):
     path, prefix = None, None
     if executable != '*':
diff --git a/pypy/module/test_lib_pypy/test_md5_extra.py b/pypy/module/test_lib_pypy/test_md5_extra.py
--- a/pypy/module/test_lib_pypy/test_md5_extra.py
+++ b/pypy/module/test_lib_pypy/test_md5_extra.py
@@ -94,7 +94,7 @@
             # interp2app doesn't work in appdirect mode
             cls.w_compare_host = staticmethod(compare_host)
         else:
-            compare_host.unwrap_spec = [str, str, str]
+            compare_host.unwrap_spec = ['bytes', 'bytes', 'text']
             cls.w_compare_host = space.wrap(gateway.interp2app(compare_host))
 
     def w_compare(self, message):
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -350,7 +350,7 @@
         space = self.space
         return space.newtext(self.filename)
 
- at unwrap_spec(name='str0')
+ at unwrap_spec(name='text0')
 def descr_new_zipimporter(space, w_type, name):
     ok = False
     parts_ends = [i for i in range(0, len(name))
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -899,7 +899,7 @@
                 successes = entry.success_counter
             globalfailures = INVALID_CACHE_ENTRY.failure_counter
             return space.wrap((failures, successes, globalfailures))
-        check.unwrap_spec = [gateway.ObjSpace, gateway.W_Root, str]
+        check.unwrap_spec = [gateway.ObjSpace, gateway.W_Root, 'text']
         cls.w_check = cls.space.wrap(gateway.interp2app(check))
 
     def test_simple(self):

From pypy.commits at gmail.com  Wed Feb 22 07:41:59 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 22 Feb 2017 04:41:59 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-2: abandon this branch
Message-ID: <58ad8717.53be190a.58996.1bd4@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-2
Changeset: r90287:6b1791c1eb9e
Date: 2017-02-22 13:41 +0100
http://bitbucket.org/pypy/pypy/changeset/6b1791c1eb9e/

Log:	abandon this branch


From pypy.commits at gmail.com  Wed Feb 22 07:43:13 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 22 Feb 2017 04:43:13 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-3: a general resume
 improvement: move the rd_count into the resumecode
Message-ID: <58ad8761.0b8d190a.a5905.1c2c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-3
Changeset: r90290:7385c2391c79
Date: 2017-02-21 21:04 +0100
http://bitbucket.org/pypy/pypy/changeset/7385c2391c79/

Log:	a general resume improvement: move the rd_count into the resumecode

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -872,10 +872,9 @@
 
 
 class ResumeGuardDescr(AbstractResumeGuardDescr):
-    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+    _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals',
                'rd_pendingfields', 'status')
     rd_numb = lltype.nullptr(NUMBERING)
-    rd_count = 0
     rd_consts = None
     rd_virtuals = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
@@ -884,7 +883,6 @@
         if isinstance(other, ResumeGuardCopiedDescr):
             other = other.prev
         assert isinstance(other, ResumeGuardDescr)
-        self.rd_count = other.rd_count
         self.rd_consts = other.rd_consts
         self.rd_pendingfields = other.rd_pendingfields
         self.rd_virtuals = other.rd_virtuals
@@ -897,7 +895,6 @@
 
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
-        self.rd_count = len(boxes)
         self.store_hash(metainterp_sd)
 
     def clone(self):
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -256,7 +256,8 @@
     def number(self, optimizer, position, trace):
         snapshot_iter = trace.get_snapshot_iter(position)
         numb_state = NumberingState(snapshot_iter.size)
-        numb_state.append_int(0) # patch later
+        numb_state.append_int(0) # patch later: size of resume section
+        numb_state.append_int(0) # patch later: number of failargs
 
         arr = snapshot_iter.vable_array
 
@@ -461,6 +462,8 @@
         self._number_virtuals(liveboxes, optimizer, num_virtuals)
         self._add_pending_fields(optimizer, pending_setfields)
 
+        numb_state.patch(1, len(liveboxes))
+
         self._add_optimizer_sections(numb_state, liveboxes, liveboxes_from_env)
         storage.rd_numb = numb_state.create_numbering()
         storage.rd_consts = self.memo.consts
@@ -929,9 +932,9 @@
     def _init(self, cpu, storage):
         self.cpu = cpu
         self.resumecodereader = resumecode.Reader(storage.rd_numb)
-        count = self.resumecodereader.next_item()
-        self.items_resume_section = count
-        self.count = storage.rd_count
+        items_resume_section = self.resumecodereader.next_item()
+        self.items_resume_section = items_resume_section
+        self.count = self.resumecodereader.next_item()
         self.consts = storage.rd_consts
 
     def _prepare(self, storage):
@@ -1075,7 +1078,7 @@
         self._init(metainterp.cpu, storage)
         self.deadframe = deadframe
         self.metainterp = metainterp
-        self.liveboxes = [None] * storage.rd_count
+        self.liveboxes = [None] * self.count
         self._prepare(storage)
 
     def consume_boxes(self, info, boxes_i, boxes_r, boxes_f):
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -2,7 +2,8 @@
 """ Resume bytecode. It goes as following:
 
   # ----- resume section
-  [total size of resume section, unencoded]
+  [total size of resume section]
+  [number of failargs]
   [<length> <virtualizable object> <numb> <numb> <numb>]    if vinfo is not None
    -OR-
   [1 <ginfo object>]                                        if ginfo is not None
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -36,7 +36,6 @@
     rd_consts = []
     rd_virtuals = None
     rd_pendingfields = None
-    rd_count = 0
 
 
 class FakeOptimizer(object):
@@ -302,12 +301,11 @@
     c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)]
     storage = Storage()
     storage.rd_consts = [c1, c2, c3]
-    numb = Numbering([tag(0, TAGBOX), tagconst(0),
+    numb = Numbering([3, tag(0, TAGBOX), tagconst(0),
                        NULLREF, tag(0, TAGBOX), tag(1, TAGBOX)] +
-                       [tagconst(1), tagconst(2)] + 
+                       [tagconst(1), tagconst(2)] +
                        [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)])
     storage.rd_numb = numb
-    storage.rd_count = 3
     #
     cpu = MyCPU([42, gcref1, -66])
     metainterp = MyMetaInterp(cpu)
@@ -348,7 +346,7 @@
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
-    numb = Numbering([tag(100, TAGINT)])
+    numb = Numbering([1, tag(100, TAGINT)])
     storage.rd_numb = numb
     #
     cpu = MyCPU([])
@@ -365,10 +363,9 @@
             return s
     class FakeStorage(object):
         rd_virtuals = [FakeVinfo(), None]
-        rd_numb = Numbering([])
+        rd_numb = Numbering([1])
         rd_consts = []
         rd_pendingfields = None
-        rd_count = 0
     class FakeMetainterp(object):
         _already_allocated_resume_virtuals = None
         cpu = None
@@ -776,12 +773,12 @@
     assert untag(tagged) == (44, TAGINT)
     tagged = memo.getconst(ConstInt(-3))
     assert untag(tagged) == (-3, TAGINT)
-    const = ConstInt(50000)
+    const = ConstInt(5000000)
     tagged = memo.getconst(const)
     index, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert memo.consts[index - TAG_CONST_OFFSET] is const
-    tagged = memo.getconst(ConstInt(50000))
+    tagged = memo.getconst(ConstInt(5000000))
     index2, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert index2 == index
@@ -861,7 +858,7 @@
     base = [0, 0, tag(0, TAGBOX), tag(1, TAGINT),
             tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)]
 
-    assert unpack_numbering(numb) == [16, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                       tag(0, TAGBOX), tag(1, TAGINT)]
     t.append(0)
     snap2 = t.create_top_snapshot(FakeJitCode("jitcode", 0), 2, Frame(env2),
@@ -875,7 +872,7 @@
     assert numb_state2.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b3: tag(2, TAGBOX)}
     assert numb_state2.liveboxes is not numb_state.liveboxes
-    assert unpack_numbering(numb2) == [16, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb2) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                        tag(0, TAGBOX), tag(3, TAGINT)]
 
     t.append(0)
@@ -897,7 +894,7 @@
     assert numb_state3.num_virtuals == 0
     
     assert numb_state3.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
-    assert unpack_numbering(numb3) == ([16, 0, 2, tag(3, TAGINT), tag(4, TAGINT),
+    assert unpack_numbering(numb3) == ([17, 0, 0, 2, tag(3, TAGINT), tag(4, TAGINT),
                                        tag(0, TAGBOX), tag(3, TAGINT)] +
                                        base + [0, 2])
 
@@ -914,7 +911,7 @@
     
     assert numb_state4.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL)}
-    assert unpack_numbering(numb4) == [16, 0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
+    assert unpack_numbering(numb4) == [17, 0, 0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
                                        tag(0, TAGBOX), tag(3, TAGINT)] + base + [0, 2]
 
     t.append(0)
@@ -933,7 +930,7 @@
 
     assert numb_state5.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)}
-    assert unpack_numbering(numb5) == [21,
+    assert unpack_numbering(numb5) == [22, 0,
         3, tag(0, TAGBOX), tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL),
         0] + base + [
         2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT)
@@ -954,14 +951,15 @@
     l = unpack_numbering(numb)
     assert l[0] == len(l)
     assert l[1] == 0
+    assert l[1] == 0
     assert l[2] == 0
     assert l[3] == 0
     assert l[4] == 0
     mapping = dict(zip(inpargs, i.inputargs))
     for i, item in enumerate(lst):
-        v, tag = untag(l[i + 5])
+        v, tag = untag(l[i + 6])
         if tag == TAGBOX:
-            assert l[i + 5] == numb_state.liveboxes[mapping[item]]
+            assert l[i + 6] == numb_state.liveboxes[mapping[item]]
         elif tag == TAGCONST:
             assert memo.consts[v].getint() == item.getint()
         elif tag == TAGINT:
@@ -1081,7 +1079,7 @@
     _next_section(reader, sys.maxint, 2**16, -65)
 
 def test_virtual_adder_memo_const_sharing():
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**23), ConstInt(-65)]
     storage, t = make_storage(b1s, b2s, b3s)
     metainterp_sd = FakeMetaInterpStaticData()
     memo = ResumeDataLoopMemo(metainterp_sd)
@@ -1091,7 +1089,7 @@
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**24), ConstInt(-65)]
     storage2, t = make_storage(b1s, b2s, b3s)
     i = t.get_iter()
     modifier2 = ResumeDataVirtualAdder(FakeOptimizer(i), storage2, storage2,
@@ -1234,7 +1232,7 @@
     liveboxes = []
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = Numbering([])
+    storage.rd_numb = Numbering([0])
     # resume
     b3t, b5t = [IntFrontendOp(0), RefFrontendOp(0)]
     b5t.setref_base(demo55o)
@@ -1305,7 +1303,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = Numbering([])
+    storage.rd_numb = Numbering([0])
     # resume
     b1t, b3t, b4t = [IntFrontendOp(0), IntFrontendOp(0), IntFrontendOp(0)]
     b1t.setint(11)
@@ -1358,7 +1356,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = Numbering([])
+    storage.rd_numb = Numbering([0])
     b4t = RefFrontendOp(0)
     newboxes = _resume_remap(liveboxes, [#b2s -- virtual
                                          b4s], b4t)
@@ -1404,7 +1402,7 @@
     modifier._add_pending_fields(FakeOptimizer(), [
         ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)])
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = Numbering([])
+    storage.rd_numb = Numbering([0])
     # resume
     demo55.next = lltype.nullptr(LLtypeMixin.NODE)
     b2t = RefFrontendOp(0)

From pypy.commits at gmail.com  Wed Feb 22 07:43:08 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 22 Feb 2017 04:43:08 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-3: store content directly
Message-ID: <58ad875c.55582e0a.d9c4c.c55c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-3
Changeset: r90288:ac91feadb8c0
Date: 2017-02-21 19:36 +0100
http://bitbucket.org/pypy/pypy/changeset/ac91feadb8c0/

Log:	store content directly

diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -81,13 +81,10 @@
 
 class Writer(object):
     def __init__(self, size=0):
-        self.current = objectmodel.newlist_hint(3 * size)
-        self.grow(size)
-        self.items = 0
+        self.current = objectmodel.newlist_hint(size)
 
     def append_short(self, item):
-        self.items += 1
-        append_numbering(self.current, item)
+        self.current.append(item)
 
     def append_int(self, item):
         short = rffi.cast(rffi.SHORT, item)
@@ -95,21 +92,19 @@
         return self.append_short(short)
 
     def create_numbering(self):
-        numb = lltype.malloc(NUMBERING, len(self.current))
-        for i, elt in enumerate(self.current):
+        final = objectmodel.newlist_hint(len(self.current) * 3)
+        for item in self.current:
+            append_numbering(final, item)
+        numb = lltype.malloc(NUMBERING, len(final))
+        for i, elt in enumerate(final):
             numb.code[i] = elt
         return numb
 
-    def grow(self, size):
-        pass
+    def patch_current_size(self, index):
+        self.patch(index, len(self.current))
 
-    def patch_current_size(self, index):
-        # mess :-(
-        assert rffi.cast(lltype.Signed, self.current[index]) == 0
-        l = []
-        append_numbering(l, self.items)
-        self.current = l + self.current[1:]
-
+    def patch(self, index, item):
+        self.current[index] = item
 
 def create_numbering(l):
     w = Writer()

From pypy.commits at gmail.com  Wed Feb 22 07:43:10 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 22 Feb 2017 04:43:10 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-3: do the right thing with
 descsr
Message-ID: <58ad875e.84472e0a.5e5fd.d174@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-3
Changeset: r90289:3b86571d166a
Date: 2017-02-21 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/3b86571d166a/

Log:	do the right thing with descsr

	(port of 96fe6ec4dec2 and 52ad37a21275)

diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
--- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -75,12 +75,12 @@
     # heap knowledge
     if optimizer.optheap:
         triples = optimizer.optheap.serialize_optheap(liveboxes_set)
+        # can only encode descrs that have a known index into
+        # metainterp_sd.all_descrs
+        triples = [triple for triple in triples if triple[1].descr_index != -1]
         numb_state.append_int(len(triples))
         for box1, descr, box2 in triples:
-            index = metainterp_sd.descrs_dct.get(descr, -1)
-            if index == -1:
-                # XXX XXX XXX fix length!
-                continue # just skip it, if the descr is not encodable
+            index = descr.descr_index
             numb_state.append_short(tag_box(box1, liveboxes_from_env, memo))
             numb_state.append_int(index)
             numb_state.append_short(tag_box(box2, liveboxes_from_env, memo))
@@ -117,8 +117,8 @@
     for i in range(length):
         tagged = reader.next_item()
         box1 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
-        tagged = reader.next_item()
-        descr = metainterp_sd.opcode_descrs[tagged]
+        index = reader.next_item()
+        descr = metainterp_sd.all_descrs[index]
         tagged = reader.next_item()
         box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
         result.append((box1, descr, box2))
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1863,9 +1863,6 @@
 
     def setup_descrs(self, descrs):
         self.opcode_descrs = descrs
-        self.descrs_dct = {}
-        for index, descr in enumerate(descrs):
-            self.descrs_dct[descr] = index
 
     def setup_indirectcalltargets(self, indirectcalltargets):
         self.indirectcalltargets = list(indirectcalltargets)
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -79,6 +79,7 @@
 def test_compile_loop():
     cpu = FakeCPU()
     staticdata = FakeMetaInterpStaticData()
+    staticdata.all_descrs = LLtypeMixin.cpu.setup_descrs()
     staticdata.cpu = cpu
     staticdata.jitlog = jl.JitLogger(cpu)
     staticdata.jitlog.trace_id = 1

From pypy.commits at gmail.com  Wed Feb 22 08:39:13 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Feb 2017 05:39:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: audioop,
 check_size also accepts size == 3 now
Message-ID: <58ad9481.421a190a.cb8af.1fa0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90291:d9c088e677ba
Date: 2017-02-22 11:34 +0100
http://bitbucket.org/pypy/pypy/changeset/d9c088e677ba/

Log:	audioop, check_size also accepts size == 3 now

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -13,9 +13,8 @@
 
 
 def _check_size(size):
-    if size != 1 and size != 2 and size != 4:
-        raise error("Size should be 1, 2 or 4")
-
+    if size < 1 or size > 4:
+        raise error("Size should be 1, 2, 3 or 4")
 
 def _check_params(length, size):
     _check_size(size)

From pypy.commits at gmail.com  Wed Feb 22 08:39:15 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Feb 2017 05:39:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: audioop support size == 3
Message-ID: <58ad9483.51072e0a.d002.602d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90292:a4c46e6650b2
Date: 2017-02-22 14:20 +0100
http://bitbucket.org/pypy/pypy/changeset/a4c46e6650b2/

Log:	audioop support size == 3

diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
--- a/lib_pypy/_audioop_build.py
+++ b/lib_pypy/_audioop_build.py
@@ -301,6 +301,32 @@
 #define CHARP(cp, i) ((signed char *)(cp+i))
 #define SHORTP(cp, i) ((short *)(cp+i))
 #define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+
+#if WORDS_BIGENDIAN
+#define GETINT24(cp, i)  (                              \
+        ((unsigned char *)(cp) + (i))[2] +              \
+        (((unsigned char *)(cp) + (i))[1] << 8) +       \
+        (((signed char *)(cp) + (i))[0] << 16) )
+#else
+#define GETINT24(cp, i)  (                              \
+        ((unsigned char *)(cp) + (i))[0] +              \
+        (((unsigned char *)(cp) + (i))[1] << 8) +       \
+        (((signed char *)(cp) + (i))[2] << 16) )
+#endif
+
+#if WORDS_BIGENDIAN
+#define SETINT24(cp, i, val)  do {                              \
+        ((unsigned char *)(cp) + (i))[2] = (int)(val);          \
+        ((unsigned char *)(cp) + (i))[1] = (int)(val) >> 8;     \
+        ((signed char *)(cp) + (i))[0] = (int)(val) >> 16;      \
+    } while (0)
+#else
+#define SETINT24(cp, i, val)  do {                              \
+        ((unsigned char *)(cp) + (i))[0] = (int)(val);          \
+        ((unsigned char *)(cp) + (i))[1] = (int)(val) >> 8;     \
+        ((signed char *)(cp) + (i))[2] = (int)(val) >> 16;      \
+    } while (0)
+#endif
 """
 
 C_SOURCE = _AUDIOOP_C_MODULE + r"""
@@ -362,6 +388,8 @@
                     cur_i[chan] = ((int)*CHARP(cp, 0)) << 24;
                 else if (size == 2)
                     cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16;
+                else if (size == 3)
+                    cur_i[chan] = ((int)GETINT24(cp, 0)) << 16;
                 else if (size == 4)
                     cur_i[chan] = (int)*LONGP(cp, 0);
                 cp += size;
@@ -384,6 +412,8 @@
                     *CHARP(ncp, 0) = (signed char)(cur_o >> 24);
                 else if (size == 2)
                     *SHORTP(ncp, 0) = (short)(cur_o >> 16);
+                else if (size == 3)
+                    SETINT24(ncp, 0, cur_o >> 8);
                 else if (size == 4)
                     *LONGP(ncp, 0) = (Py_Int32)(cur_o);
                 ncp += size;
@@ -407,6 +437,7 @@
     for ( i=0; i < len; i += size ) {
         if ( size == 1 )      val = (int)*CHARP(cp, i);
         else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 3 ) val = (int)GETINT24(cp, i);
         else if ( size == 4 ) val = (int)*LONGP(cp, i);
 
         fval = (double)val*fac1;
@@ -417,10 +448,12 @@
 
         if ( size == 1 )      *CHARP(ncp, i*2) = (signed char)val1;
         else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1;
+        else if ( size == 3 ) SETINT24(ncp, i*2, val1);
         else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1;
 
         if ( size == 1 )      *CHARP(ncp, i*2+1) = (signed char)val2;
         else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2;
+        else if ( size == 3 ) SETINT24(ncp, i*2, val1);
         else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2;
     }
 }
@@ -437,10 +470,12 @@
     for ( i=0; i < len1; i += size ) {
         if ( size == 1 )      val1 = (int)*CHARP(cp1, i);
         else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i);
+        else if ( size == 3 ) val1 = (int)GETINT24(cp1, i);
         else if ( size == 4 ) val1 = (int)*LONGP(cp1, i);
 
         if ( size == 1 )      val2 = (int)*CHARP(cp2, i);
         else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i);
+        else if ( size == 3 ) val2 = (int)GETINT24(cp2, i);
         else if ( size == 4 ) val2 = (int)*LONGP(cp2, i);
 
         if (size < 4) {
@@ -459,6 +494,7 @@
 
         if ( size == 1 )      *CHARP(ncp, i) = (signed char)newval;
         else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval;
+        else if ( size == 3 ) SETINT24(ncp, i, newval);
         else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval;
     }
 }
@@ -479,6 +515,7 @@
     for ( i=0; i < len; i += size ) {
         if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
         else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 3 ) val = ((int)GETINT24(cp, i)) >> 8;
         else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
 
         /* Step 1 - compute difference with previous value */
@@ -608,6 +645,7 @@
         /* Step 6 - Output value */
         if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
         else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 3 ) SETINT24(ncp, i, valpred << 8);
         else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
     }
     state[0] = valpred;
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -1,10 +1,11 @@
+import sys
 import builtins
 import math
 import struct
-from fractions import gcd
+from math import gcd
 from _audioop_cffi import ffi, lib
 
-
+BIG_ENDIAN = sys.byteorder != 'little'
 _buffer = memoryview
 
 
@@ -47,18 +48,49 @@
         return "b" if signed else "B"
     elif size == 2:
         return "h" if signed else "H"
+    elif size == 3:
+        raise NotImplementedError
     elif size == 4:
         return "i" if signed else "I"
 
+def _unpack_int24(buf):
+    if BIG_ENDIAN:
+        val = (buf[2] & 0xff) | \
+               ((buf[1] & 0xff) << 8) | \
+               ((buf[0] & 0xff) << 16)
+    else:
+        val = (buf[0] & 0xff) | \
+               ((buf[1] & 0xff) << 8) | \
+               ((buf[2] & 0xff) << 16)
+    if val & 0x800000:
+        val = val - 0x1000000
+    return val
+
+def _pack_int24(into, off, val):
+    buf = _buffer(into)
+    if BIG_ENDIAN:
+        buf[off+2] = val & 0xff
+        buf[off+1] = (val >> 8) & 0xff
+        buf[off+0] = (val >> 16) & 0xff
+    else:
+        buf[off+0] = val & 0xff
+        buf[off+1] = (val >> 8) & 0xff
+        buf[off+2] = (val >> 16) & 0xff
 
 def _get_sample(cp, size, i, signed=True):
-    fmt = _struct_format(size, signed)
     start = i * size
     end = start + size
-    return struct.unpack_from(fmt, _buffer(cp)[start:end])[0]
+    chars = _buffer(cp)[start:end]
+    if size == 3:
+        return _unpack_int24(chars)
+    fmt = _struct_format(size, signed)
+    return struct.unpack_from(fmt, chars)[0]
 
 
 def _put_sample(cp, size, i, val, signed=True):
+    if size == 3:
+        _pack_int24(cp, i*size, val)
+        return
     fmt = _struct_format(size, signed)
     struct.pack_into(fmt, cp, i * size, val)
 
@@ -108,8 +140,17 @@
     else:
         return val % (2**bits)
 
+def _check_bytes(cp):
+    # we have no argument clinic
+    try:
+        memoryview(cp)
+    except TypeError:
+        raise TypeError("a bytes-like object is required, not '%s'" % \
+                str(type(cp)))
+
 
 def getsample(cp, size, i):
+    # _check_bytes checked in _get_sample
     _check_params(len(cp), size)
     if not (0 <= i < len(cp) // size):
         raise error("Index out of range")
@@ -249,6 +290,7 @@
 
 
 def avgpp(cp, size):
+    _check_bytes(cp)
     _check_params(len(cp), size)
     sample_count = _sample_count(cp, size)
     if sample_count <= 2:
@@ -415,6 +457,7 @@
 
 
 def lin2lin(cp, size, size2):
+    _check_bytes(cp)
     _check_params(len(cp), size)
     _check_size(size2)
 
@@ -504,6 +547,8 @@
             yield sample << 8
         elif size == 2:
             yield sample
+        elif size == 3:
+            yield sample >> 8
         elif size == 4:
             yield sample >> 16
 
@@ -513,6 +558,8 @@
         sample >>= 8
     elif size == 2:
         pass
+    elif size == 3:
+        sample <<= 8
     elif size == 4:
         sample <<= 16
     _put_sample(result, size, i, sample)

From pypy.commits at gmail.com  Wed Feb 22 08:39:17 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Feb 2017 05:39:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: some last tweaks to pass the test audioop
 suite
Message-ID: <58ad9485.5312190a.c5163.1f4e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90293:f533d9936aca
Date: 2017-02-22 14:38 +0100
http://bitbucket.org/pypy/pypy/changeset/f533d9936aca/

Log:	some last tweaks to pass the test audioop suite

diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
--- a/lib_pypy/_audioop_build.py
+++ b/lib_pypy/_audioop_build.py
@@ -389,7 +389,7 @@
                 else if (size == 2)
                     cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16;
                 else if (size == 3)
-                    cur_i[chan] = ((int)GETINT24(cp, 0)) << 16;
+                    cur_i[chan] = ((int)GETINT24(cp, 0)) << 8;
                 else if (size == 4)
                     cur_i[chan] = (int)*LONGP(cp, 0);
                 cp += size;
@@ -453,7 +453,7 @@
 
         if ( size == 1 )      *CHARP(ncp, i*2+1) = (signed char)val2;
         else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2;
-        else if ( size == 3 ) SETINT24(ncp, i*2, val1);
+        else if ( size == 3 ) SETINT24(ncp, i*2+3, val2);
         else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2;
     }
 }
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -104,6 +104,10 @@
         return 0x7fff
     elif size == 2:
         return 0xffff
+    elif signed and size == 3:
+        return 0x7fffff
+    elif size == 3:
+        return 0xffffff
     elif signed and size == 4:
         return 0x7fffffff
     elif size == 4:
@@ -117,6 +121,8 @@
         return -0x80
     elif size == 2:
         return -0x8000
+    elif size == 3:
+        return -0x800000
     elif size == 4:
         return -0x80000000
 
@@ -474,10 +480,14 @@
             sample <<= 24
         elif size == 2:
             sample <<= 16
+        elif size == 3:
+            sample <<= 8
         if size2 == 1:
             sample >>= 24
         elif size2 == 2:
             sample >>= 16
+        elif size2 == 3:
+            sample >>= 8
         sample = _overflow(sample, size2)
         _put_sample(result, size2, i, sample)
 
@@ -531,7 +541,8 @@
     nbytes = ceiling * bytes_per_frame
 
     rv = ffi.new("char[]", nbytes)
-    trim_index = lib.ratecv(rv, cp, frame_count, size,
+    cpbuf = ffi.from_buffer(cp)
+    trim_index = lib.ratecv(rv, cpbuf, frame_count, size,
                             nchannels, inrate, outrate,
                             state_d, prev_i, cur_i,
                             weightA, weightB)
@@ -606,7 +617,8 @@
     state = _check_state(state)
     rv = ffi.new("unsigned char[]", len(cp) // size // 2)
     state_ptr = ffi.new("int[]", state)
-    lib.lin2adcpm(rv, cp, len(cp), size, state_ptr)
+    cpbuf = ffi.cast("unsigned char*", ffi.from_buffer(cp))
+    lib.lin2adcpm(rv, cpbuf, len(cp), size, state_ptr)
     return ffi.buffer(rv)[:], tuple(state_ptr)
 
 
@@ -615,7 +627,8 @@
     state = _check_state(state)
     rv = ffi.new("unsigned char[]", len(cp) * size * 2)
     state_ptr = ffi.new("int[]", state)
-    lib.adcpm2lin(rv, ffi.from_buffer(cp), len(cp), size, state_ptr)
+    cpbuf = ffi.cast("unsigned char*", ffi.from_buffer(cp))
+    lib.adcpm2lin(rv, cpbuf, len(cp), size, state_ptr)
     return ffi.buffer(rv)[:], tuple(state_ptr)
 
 def byteswap(cp, size):

From pypy.commits at gmail.com  Wed Feb 22 08:58:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 05:58:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Remove some more space.str_w and 'str0'
Message-ID: <58ad9914.1c152e0a.d64cd.20c1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90295:fbd3692ba3cd
Date: 2017-02-22 14:31 +0100
http://bitbucket.org/pypy/pypy/changeset/fbd3692ba3cd/

Log:	Remove some more space.str_w and 'str0'

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -362,7 +362,7 @@
             raise
         # Unicode/ValueError in literal: turn into SyntaxError
         e.normalize_exception(space)
-        errmsg = space.str_w(space.str(e.get_w_value(space)))
+        errmsg = space.text_w(space.str(e.get_w_value(space)))
         raise astbuilder.error('(%s) %s' % (kind, errmsg), atom_node)
 
     if not fmode and len(joined_pieces) == 1:   # <= the common path
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -300,7 +300,7 @@
 
     def descr_set__name__(self, space, w_name):
         if space.isinstance_w(w_name, space.w_unicode):
-            self._name = space.str_w(w_name)
+            self._name = space.text_w(w_name)
         else:
             raise oefmt(space.w_TypeError,
                         "__name__ must be set to a string object")
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -376,7 +376,7 @@
 
     @unwrap_spec(argcount=int, kwonlyargcount=int, nlocals=int, stacksize=int, flags=int,
                  codestring='bytes',
-                 filename='str0', name='text', firstlineno=int,
+                 filename='fsencode', name='text', firstlineno=int,
                  lnotab='bytes', magic=int)
     def descr_code__new__(space, w_subtype,
                           argcount, kwonlyargcount, nlocals, stacksize, flags,
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -365,7 +365,7 @@
         obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
         start = space.int_w(space.getattr(w_exc, space.newtext('start')))
         w_end = space.getattr(w_exc, space.newtext('end'))
-        encoding = space.str_w(space.getattr(w_exc, space.newtext('encoding')))
+        encoding = space.text_w(space.getattr(w_exc, space.newtext('encoding')))
         bytelength, code = get_standard_encoding(encoding)
         if code == ENC_UNKNOWN:
             # Not supported, fail with original exception
@@ -403,7 +403,7 @@
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
         start = space.int_w(space.getattr(w_exc, space.newtext('start')))
         obj = space.bytes_w(space.getattr(w_exc, space.newtext('object')))
-        encoding = space.str_w(space.getattr(w_exc, space.newtext('encoding')))
+        encoding = space.text_w(space.getattr(w_exc, space.newtext('encoding')))
         bytelength, code = get_standard_encoding(encoding)
         ch = 0
         # Try decoding a single surrogate character. If there are more,
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -215,7 +215,7 @@
         return rffi.cast(SEM_T, space.int_w(w_handle))
 
     def semaphore_unlink(space, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         try:
             sem_unlink(name)
         except OSError as e:
@@ -550,7 +550,7 @@
     def _finalize_(self):
         delete_semaphore(self.handle)
 
- at unwrap_spec(kind=int, value=int, maxvalue=int, name=str, unlink=int)
+ at unwrap_spec(kind=int, value=int, maxvalue=int, name='text', unlink=int)
 def descr_new(space, w_subtype, kind, value, maxvalue, name, unlink):
     if kind != RECURSIVE_MUTEX and kind != SEMAPHORE:
         raise oefmt(space.w_ValueError, "unrecognized kind")
diff --git a/pypy/module/_rawffi/alt/interp_struct.py b/pypy/module/_rawffi/alt/interp_struct.py
--- a/pypy/module/_rawffi/alt/interp_struct.py
+++ b/pypy/module/_rawffi/alt/interp_struct.py
@@ -102,7 +102,7 @@
         return W__StructInstance(self, allocate=False, autofree=True, rawmem=rawmem)
 
     def get_type_and_offset_for_field(self, space, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         try:
             return self._get_type_and_offset_for_field(space, name)
         except KeyError:
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -132,7 +132,7 @@
 
     Get host and port for a sockaddr."""
     try:
-        host = space.str_w((space.getitem(w_sockaddr, space.newint(0))))
+        host = space.text_w((space.getitem(w_sockaddr, space.newint(0))))
         port = str(space.int_w(space.getitem(w_sockaddr, space.newint(1))))
         lst = rsocket.getaddrinfo(host, port, rsocket.AF_UNSPEC,
                                   rsocket.SOCK_DGRAM, 0,
@@ -310,7 +310,7 @@
     elif space.isinstance_w(w_port, space.w_bytes):
         port = space.bytes_w(w_port)
     elif space.isinstance_w(w_port, space.w_unicode):
-        port = space.str_w(w_port)
+        port = space.text_w(w_port)
     else:
         raise oefmt(space.w_TypeError,
                     "getaddrinfo() argument 2 must be integer or string")
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -160,8 +160,8 @@
         if space.isinstance_w(w_string, space.w_unicode):
             unicodestr = space.unicode_w(w_string)
             length = len(unicodestr)
-        elif space.isinstance_w(w_string, space.w_str):
-            string = space.str_w(w_string)
+        elif space.isinstance_w(w_string, space.w_bytes):
+            string = space.bytes_w(w_string)
             length = len(string)
         else:
             buf = space.readbuf_w(w_string)
diff --git a/pypy/module/array/reconstructor.py b/pypy/module/array/reconstructor.py
--- a/pypy/module/array/reconstructor.py
+++ b/pypy/module/array/reconstructor.py
@@ -71,7 +71,7 @@
 MACHINE_FORMAT_CODE_MAX = max(format_descriptors)
 
 
- at unwrap_spec(typecode=str, mformat_code=int)
+ at unwrap_spec(typecode='text', mformat_code=int)
 def array_reconstructor(space, w_cls, typecode, mformat_code, w_items):
     # Fast path: machine format code corresponds to the
     # platform-independent typecode.
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -135,7 +135,7 @@
                   consts=space.fixedview(w_consts),
                   names=unwrap_list_of_texts(space, w_names),
                   varnames=unwrap_list_of_texts(space, w_varnames),
-                  filename=space.str0_w(w_filename),
+                  filename=space.fsencode_w(w_filename),
                   name=space.text_w(w_funcname),
                   firstlineno=rffi.cast(lltype.Signed, firstlineno),
                   lnotab=space.bytes_w(w_lnotab),
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -52,7 +52,7 @@
     except ValueError:
         w_s = w_obj.descr_tobytes(space)
         view.c_obj = make_ref(space, w_s)
-        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.bytes_w(w_s),
                                              track_allocation=False))
         rffi.setintfield(view, 'c_readonly', 1)
 
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -358,7 +358,7 @@
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.str_w(w_reason)
+        space.text_w(w_reason)
         # assign attributes
         self.w_object = w_object
         self.w_start = w_start
@@ -908,11 +908,11 @@
             w_bytes = space.newbytes(space.bufferstr_w(w_object))
         else:
             w_bytes = w_object
-        space.str_w(w_encoding)
+        space.text_w(w_encoding)
         space.bytes_w(w_bytes)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.str_w(w_reason)
+        space.text_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_bytes
@@ -1001,11 +1001,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.str_w(w_encoding)
+        space.text_w(w_encoding)
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.str_w(w_reason)
+        space.text_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py
--- a/pypy/module/parser/pyparser.py
+++ b/pypy/module/parser/pyparser.py
@@ -49,7 +49,7 @@
         return self._build_app_tree(space, self.tree, space.newlist,
                                     line_info, col_info)
 
-    @unwrap_spec(filename='str0')
+    @unwrap_spec(filename='fsencode')
     def descr_compile(self, space, filename="<syntax-tree>"):
         info = pyparse.CompileInfo(filename, self.mode)
         try:
diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -19,7 +19,7 @@
     if space.is_none(w_path):
         w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
-        path_bytes = space.str0_w(w_path)
+        path_bytes = space.bytes0_w(w_path)
         result_is_bytes = True
     else:
         path_bytes = space.fsencode_w(w_path)
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -208,7 +208,7 @@
     __length_hint__=interp2app(W_UnpackIter.descr_length_hint)
 )
 
- at unwrap_spec(format=str)
+ at unwrap_spec(format='text')
 def iter_unpack(space, format, w_buffer):
     w_struct = W_Struct(space, format)
     return W_UnpackIter(space, w_struct, w_buffer)
diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py
--- a/pypy/module/sys/interp_encoding.py
+++ b/pypy/module/sys/interp_encoding.py
@@ -34,7 +34,7 @@
                                               space.newtext(loc_codeset))
                     if space.is_true(w_res):
                         w_name = space.getattr(w_res, space.newtext('name'))
-                        encoding = space.str_w(w_name)
+                        encoding = space.text_w(w_name)
             finally:
                 rlocale.setlocale(rlocale.LC_CTYPE, oldlocale)
         except rlocale.LocaleError:
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -487,8 +487,8 @@
                     self.std_wp(s)
                 return
             if not do_unicode:
-                if space.isinstance_w(w_value, space.w_str):
-                    s = space.str_w(w_value)
+                if space.isinstance_w(w_value, space.w_bytes):
+                    s = space.bytes_w(w_value)
                 elif space.isinstance_w(w_value, space.w_bytearray):
                     s = w_value.buffer_w(space, 0).as_str()
                 else:
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -58,7 +58,7 @@
         return space.newlong_from_rbigint(b)
 
     @staticmethod
-    @unwrap_spec(byteorder=str, signed=bool)
+    @unwrap_spec(byteorder='text', signed=bool)
     def descr_from_bytes(space, w_inttype, w_obj, byteorder, signed=False):
         """int.from_bytes(bytes, byteorder, *, signed=False) -> int
 
@@ -97,7 +97,7 @@
             w_obj = space.call_function(w_inttype, w_obj)
         return w_obj
 
-    @unwrap_spec(nbytes=int, byteorder=str, signed=bool)
+    @unwrap_spec(nbytes=int, byteorder='text', signed=bool)
     def descr_to_bytes(self, space, nbytes, byteorder, signed=False):
         """to_bytes(...)
         int.to_bytes(length, byteorder, *, signed=False) -> bytes
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -529,7 +529,7 @@
             raise oefmt(space.w_TypeError,
                         "memoryview: format argument must be a string")
 
-        fmt = space.str_w(w_format)
+        fmt = space.text_w(w_format)
         buf = self.buf
         ndim = 1
 

From pypy.commits at gmail.com  Wed Feb 22 08:58:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 05:58:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58ad9912.8c2b190a.3ecf2.2021@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90294:7639500dbd0e
Date: 2017-02-22 14:14 +0100
http://bitbucket.org/pypy/pypy/changeset/7639500dbd0e/

Log:	hg merge default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1584,7 +1584,7 @@
     def text_or_None_w(self, w_obj):
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
-    #@not_rpython    BACKCOMPAT: should be replaced with bytes_w or text_w
+    @not_rpython    # tests only; should be replaced with bytes_w or text_w
     def str_w(self, w_obj):
         """
         if w_obj is unicode, call text_w() (i.e., return the UTF-8-nosg
@@ -1602,7 +1602,7 @@
     def bytes_w(self, w_obj):
         return w_obj.bytes_w(self)
 
-    #@not_rpython    BACKCOMPAT
+    @not_rpython    # tests only; should be replaced with bytes0_w or text0_w
     def str0_w(self, w_obj):
         "Like str_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -162,9 +162,6 @@
     def visit_text_or_None(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
-    def visit_str0(self, el, app_sig):
-        self.checked_space_method(el, app_sig)
-
     def visit_bytes(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -314,9 +311,6 @@
     def visit_text_or_None(self, typ):
         self.run_args.append("space.text_or_None_w(%s)" % (self.scopenext(),))
 
-    def visit_str0(self, typ):
-        self.run_args.append("space.str0_w(%s)" % (self.scopenext(),))
-
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
 
@@ -485,9 +479,6 @@
     def visit_text_or_None(self, typ):
         self.unwrap.append("space.text_or_None_w(%s)" % (self.nextarg(),))
 
-    def visit_str0(self, typ):
-        self.unwrap.append("space.str0_w(%s)" % (self.nextarg(),))
-
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
 
@@ -648,6 +639,8 @@
                              "the name of an argument of the following "
                              "function" % (name,))
 
+    assert str not in unwrap_spec   # use 'text' or 'bytes' instead of str
+
     return unwrap_spec
 
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -49,7 +49,7 @@
             space.call_method(self.w_dict, 'update', self.w_initialdict)
 
         for w_submodule in self.submodules_w:
-            name = space.str0_w(w_submodule.w_name)
+            name = space.text0_w(w_submodule.w_name)
             space.setitem(self.w_dict, space.newtext(name.split(".")[-1]), w_submodule)
             space.getbuiltinmodule(name)
 
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -37,7 +37,7 @@
 
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
-        modulename = self.space.str0_w(self.w_name)
+        modulename = self.space.text0_w(self.w_name)
         self.space.builtin_modules[modulename] = self
 
     def setup_after_space_initialization(self):
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -384,7 +384,7 @@
             return space.wrap(s0+s1)
         app_g3_ss = gateway.interp2app_temp(g3_ss,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      str, 'str_or_None'])
+                                                      'text', 'str_or_None'])
         w_app_g3_ss = space.wrap(app_g3_ss)
         assert self.space.eq_w(
             space.call(w_app_g3_ss,
@@ -530,7 +530,7 @@
 
         app_g3_s = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      str])
+                                                      'text'])
         w_app_g3_s = space.wrap(app_g3_s)
         assert space.eq_w(space.call_function(w_app_g3_s,w("foo")),w("foo"))
         raises(gateway.OperationError,space.call_function,w_app_g3_s,w(None))
diff --git a/pypy/module/__pypy__/interp_os.py b/pypy/module/__pypy__/interp_os.py
--- a/pypy/module/__pypy__/interp_os.py
+++ b/pypy/module/__pypy__/interp_os.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 
 
- at unwrap_spec(name='str0')
+ at unwrap_spec(name='text0')
 def real_getenv(space, name):
     """Get an OS environment value skipping Python cache"""
     return space.newtext_or_none(os.environ.get(name))
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -6,7 +6,7 @@
 import pypy.module.cpyext.api     # side-effect of pre-importing it
 
 
- at unwrap_spec(cdef=str, module_name=str, source=str, packed=int)
+ at unwrap_spec(cdef='text', module_name='text', source='text', packed=int)
 def prepare(space, cdef, module_name, source, w_includes=None,
             w_extra_source=None, w_min_version=None, packed=False):
     try:
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -114,7 +114,7 @@
 # __________________________________________________________
 # functions for the "win32" namespace
 
- at unwrap_spec(name=str, openmode=r_uint, pipemode=r_uint, maxinstances=r_uint,
+ at unwrap_spec(name='text', openmode=r_uint, pipemode=r_uint, maxinstances=r_uint,
              outputsize=r_uint, inputsize=r_uint, timeout=r_uint)
 def CreateNamedPipe(space, name, openmode, pipemode, maxinstances,
                     outputsize, inputsize, timeout, w_security):
@@ -161,13 +161,13 @@
         lltype.free(state, flavor='raw')
         lltype.free(statep, flavor='raw')
 
- at unwrap_spec(name=str, timeout=r_uint)
+ at unwrap_spec(name='text', timeout=r_uint)
 def WaitNamedPipe(space, name, timeout):
     # Careful: zero means "default value specified by CreateNamedPipe()"
     if not _WaitNamedPipe(name, timeout):
         raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
- at unwrap_spec(filename=str, access=r_uint, share=r_uint,
+ at unwrap_spec(filename='fsencode', access=r_uint, share=r_uint,
              disposition=r_uint, flags=r_uint)
 def CreateFile(space, filename, access, share, w_security,
                disposition, flags, w_templatefile):
diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py
--- a/pypy/module/_rawffi/alt/test/test_struct.py
+++ b/pypy/module/_rawffi/alt/test/test_struct.py
@@ -57,7 +57,7 @@
         if cls.runappdirect:
             cls.w_read_raw_mem = cls.read_raw_mem
         else:
-            @unwrap_spec(addr=int, typename=str, length=int)
+            @unwrap_spec(addr=int, typename='text', length=int)
             def read_raw_mem_w(space, addr, typename, length):
                 return space.wrap(cls.read_raw_mem(addr, typename, length))
             cls.w_read_raw_mem = cls.space.wrap(interp2app(read_raw_mem_w))
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -76,7 +76,7 @@
             cls.w_create_temp_file = cls.space.wrap(
                 gateway.interp2app(create_temp_file_w))
 
-            @gateway.unwrap_spec(data=bytes)
+            @gateway.unwrap_spec(data='bytes')
             def decompress_w(space, data):
                 return space.newbytes(decompress(cls, data))
             cls.w_decompress = cls.space.wrap(gateway.interp2app(decompress_w))
diff --git a/pypy/module/cppyy/test/test_crossing.py b/pypy/module/cppyy/test/test_crossing.py
--- a/pypy/module/cppyy/test/test_crossing.py
+++ b/pypy/module/cppyy/test/test_crossing.py
@@ -78,7 +78,7 @@
             import ctypes, cppyy""")    # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
-        @unwrap_spec(name=str, init=str, body=str)
+        @unwrap_spec(name='text', init='text', body='text')
         def create_cdll(space, name, init, body):
             # the following is loosely from test_cpyext.py import_module; it
             # is copied here to be able to tweak the call to
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -289,7 +289,7 @@
         if self.runappdirect:
             return
 
-        @unwrap_spec(name=str)
+        @unwrap_spec(name='text')
         def compile_module(space, name,
                            w_source_files=None,
                            w_source_strings=None):
@@ -315,7 +315,7 @@
 
             return space.wrap(pydname)
 
-        @unwrap_spec(name=str, init='str_or_None', body=str,
+        @unwrap_spec(name='text', init='str_or_None', body='text',
                      filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
         def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
@@ -327,12 +327,12 @@
             return w_result
 
 
-        @unwrap_spec(mod=str, name=str)
+        @unwrap_spec(mod='text', name='text')
         def load_module(space, mod, name):
             return self.sys_info.load_module(mod, name)
 
-        @unwrap_spec(modname=str, prologue=str,
-                             more_init=str, PY_SSIZE_T_CLEAN=bool)
+        @unwrap_spec(modname='text', prologue='text',
+                             more_init='text', PY_SSIZE_T_CLEAN=bool)
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -79,7 +79,7 @@
 
 # ____________________________________________________________
 
- at unwrap_spec(filename='str0')
+ at unwrap_spec(filename='fsencode')
 def dump_heap_stats(space, filename):
     tb = rgc._heap_stats()
     if not tb:
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -56,7 +56,7 @@
 
 def create_builtin(space, w_spec):
     w_name = space.getattr(w_spec, space.newtext("name"))
-    name = space.str0_w(w_name)
+    name = space.text0_w(w_name)
     # force_init is needed to make reload actually reload instead of just
     # using the already-present module in sys.modules.
 
diff --git a/pypy/module/test_lib_pypy/test_md5_extra.py b/pypy/module/test_lib_pypy/test_md5_extra.py
--- a/pypy/module/test_lib_pypy/test_md5_extra.py
+++ b/pypy/module/test_lib_pypy/test_md5_extra.py
@@ -101,7 +101,7 @@
             import py
             py.test.skip('Unavailable under py3 runappdirect')
         else:
-            compare_host.unwrap_spec = [str, str, str]
+            compare_host.unwrap_spec = ['bytes', 'bytes', 'text']
             cls.w_compare_host = space.wrap(gateway.interp2app(compare_host))
 
     def w_compare(self, message):
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -910,7 +910,7 @@
                 successes = entry.success_counter
             globalfailures = INVALID_CACHE_ENTRY.failure_counter
             return space.wrap((failures, successes, globalfailures))
-        check.unwrap_spec = [gateway.ObjSpace, gateway.W_Root, str]
+        check.unwrap_spec = [gateway.ObjSpace, gateway.W_Root, 'text']
         cls.w_check = cls.space.wrap(gateway.interp2app(check))
 
     def test_simple(self):

From pypy.commits at gmail.com  Wed Feb 22 08:58:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 05:58:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <58ad9917.4395190a.2afe7.2081@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90296:d15adb9f2806
Date: 2017-02-22 14:57 +0100
http://bitbucket.org/pypy/pypy/changeset/d15adb9f2806/

Log:	merge heads

diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
--- a/lib_pypy/_audioop_build.py
+++ b/lib_pypy/_audioop_build.py
@@ -301,6 +301,32 @@
 #define CHARP(cp, i) ((signed char *)(cp+i))
 #define SHORTP(cp, i) ((short *)(cp+i))
 #define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+
+#if WORDS_BIGENDIAN
+#define GETINT24(cp, i)  (                              \
+        ((unsigned char *)(cp) + (i))[2] +              \
+        (((unsigned char *)(cp) + (i))[1] << 8) +       \
+        (((signed char *)(cp) + (i))[0] << 16) )
+#else
+#define GETINT24(cp, i)  (                              \
+        ((unsigned char *)(cp) + (i))[0] +              \
+        (((unsigned char *)(cp) + (i))[1] << 8) +       \
+        (((signed char *)(cp) + (i))[2] << 16) )
+#endif
+
+#if WORDS_BIGENDIAN
+#define SETINT24(cp, i, val)  do {                              \
+        ((unsigned char *)(cp) + (i))[2] = (int)(val);          \
+        ((unsigned char *)(cp) + (i))[1] = (int)(val) >> 8;     \
+        ((signed char *)(cp) + (i))[0] = (int)(val) >> 16;      \
+    } while (0)
+#else
+#define SETINT24(cp, i, val)  do {                              \
+        ((unsigned char *)(cp) + (i))[0] = (int)(val);          \
+        ((unsigned char *)(cp) + (i))[1] = (int)(val) >> 8;     \
+        ((signed char *)(cp) + (i))[2] = (int)(val) >> 16;      \
+    } while (0)
+#endif
 """
 
 C_SOURCE = _AUDIOOP_C_MODULE + r"""
@@ -362,6 +388,8 @@
                     cur_i[chan] = ((int)*CHARP(cp, 0)) << 24;
                 else if (size == 2)
                     cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16;
+                else if (size == 3)
+                    cur_i[chan] = ((int)GETINT24(cp, 0)) << 8;
                 else if (size == 4)
                     cur_i[chan] = (int)*LONGP(cp, 0);
                 cp += size;
@@ -384,6 +412,8 @@
                     *CHARP(ncp, 0) = (signed char)(cur_o >> 24);
                 else if (size == 2)
                     *SHORTP(ncp, 0) = (short)(cur_o >> 16);
+                else if (size == 3)
+                    SETINT24(ncp, 0, cur_o >> 8);
                 else if (size == 4)
                     *LONGP(ncp, 0) = (Py_Int32)(cur_o);
                 ncp += size;
@@ -407,6 +437,7 @@
     for ( i=0; i < len; i += size ) {
         if ( size == 1 )      val = (int)*CHARP(cp, i);
         else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 3 ) val = (int)GETINT24(cp, i);
         else if ( size == 4 ) val = (int)*LONGP(cp, i);
 
         fval = (double)val*fac1;
@@ -417,10 +448,12 @@
 
         if ( size == 1 )      *CHARP(ncp, i*2) = (signed char)val1;
         else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1;
+        else if ( size == 3 ) SETINT24(ncp, i*2, val1);
         else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1;
 
         if ( size == 1 )      *CHARP(ncp, i*2+1) = (signed char)val2;
         else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2;
+        else if ( size == 3 ) SETINT24(ncp, i*2+3, val2);
         else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2;
     }
 }
@@ -437,10 +470,12 @@
     for ( i=0; i < len1; i += size ) {
         if ( size == 1 )      val1 = (int)*CHARP(cp1, i);
         else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i);
+        else if ( size == 3 ) val1 = (int)GETINT24(cp1, i);
         else if ( size == 4 ) val1 = (int)*LONGP(cp1, i);
 
         if ( size == 1 )      val2 = (int)*CHARP(cp2, i);
         else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i);
+        else if ( size == 3 ) val2 = (int)GETINT24(cp2, i);
         else if ( size == 4 ) val2 = (int)*LONGP(cp2, i);
 
         if (size < 4) {
@@ -459,6 +494,7 @@
 
         if ( size == 1 )      *CHARP(ncp, i) = (signed char)newval;
         else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval;
+        else if ( size == 3 ) SETINT24(ncp, i, newval);
         else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval;
     }
 }
@@ -479,6 +515,7 @@
     for ( i=0; i < len; i += size ) {
         if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
         else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 3 ) val = ((int)GETINT24(cp, i)) >> 8;
         else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
 
         /* Step 1 - compute difference with previous value */
@@ -608,6 +645,7 @@
         /* Step 6 - Output value */
         if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
         else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 3 ) SETINT24(ncp, i, valpred << 8);
         else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
     }
     state[0] = valpred;
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -1,10 +1,11 @@
+import sys
 import builtins
 import math
 import struct
-from fractions import gcd
+from math import gcd
 from _audioop_cffi import ffi, lib
 
-
+BIG_ENDIAN = sys.byteorder != 'little'
 _buffer = memoryview
 
 
@@ -13,9 +14,8 @@
 
 
 def _check_size(size):
-    if size != 1 and size != 2 and size != 4:
-        raise error("Size should be 1, 2 or 4")
-
+    if size < 1 or size > 4:
+        raise error("Size should be 1, 2, 3 or 4")
 
 def _check_params(length, size):
     _check_size(size)
@@ -48,18 +48,49 @@
         return "b" if signed else "B"
     elif size == 2:
         return "h" if signed else "H"
+    elif size == 3:
+        raise NotImplementedError
     elif size == 4:
         return "i" if signed else "I"
 
+def _unpack_int24(buf):
+    if BIG_ENDIAN:
+        val = (buf[2] & 0xff) | \
+               ((buf[1] & 0xff) << 8) | \
+               ((buf[0] & 0xff) << 16)
+    else:
+        val = (buf[0] & 0xff) | \
+               ((buf[1] & 0xff) << 8) | \
+               ((buf[2] & 0xff) << 16)
+    if val & 0x800000:
+        val = val - 0x1000000
+    return val
+
+def _pack_int24(into, off, val):
+    buf = _buffer(into)
+    if BIG_ENDIAN:
+        buf[off+2] = val & 0xff
+        buf[off+1] = (val >> 8) & 0xff
+        buf[off+0] = (val >> 16) & 0xff
+    else:
+        buf[off+0] = val & 0xff
+        buf[off+1] = (val >> 8) & 0xff
+        buf[off+2] = (val >> 16) & 0xff
 
 def _get_sample(cp, size, i, signed=True):
-    fmt = _struct_format(size, signed)
     start = i * size
     end = start + size
-    return struct.unpack_from(fmt, _buffer(cp)[start:end])[0]
+    chars = _buffer(cp)[start:end]
+    if size == 3:
+        return _unpack_int24(chars)
+    fmt = _struct_format(size, signed)
+    return struct.unpack_from(fmt, chars)[0]
 
 
 def _put_sample(cp, size, i, val, signed=True):
+    if size == 3:
+        _pack_int24(cp, i*size, val)
+        return
     fmt = _struct_format(size, signed)
     struct.pack_into(fmt, cp, i * size, val)
 
@@ -73,6 +104,10 @@
         return 0x7fff
     elif size == 2:
         return 0xffff
+    elif signed and size == 3:
+        return 0x7fffff
+    elif size == 3:
+        return 0xffffff
     elif signed and size == 4:
         return 0x7fffffff
     elif size == 4:
@@ -86,6 +121,8 @@
         return -0x80
     elif size == 2:
         return -0x8000
+    elif size == 3:
+        return -0x800000
     elif size == 4:
         return -0x80000000
 
@@ -109,8 +146,17 @@
     else:
         return val % (2**bits)
 
+def _check_bytes(cp):
+    # we have no argument clinic
+    try:
+        memoryview(cp)
+    except TypeError:
+        raise TypeError("a bytes-like object is required, not '%s'" % \
+                str(type(cp)))
+
 
 def getsample(cp, size, i):
+    # _check_bytes checked in _get_sample
     _check_params(len(cp), size)
     if not (0 <= i < len(cp) // size):
         raise error("Index out of range")
@@ -250,6 +296,7 @@
 
 
 def avgpp(cp, size):
+    _check_bytes(cp)
     _check_params(len(cp), size)
     sample_count = _sample_count(cp, size)
     if sample_count <= 2:
@@ -416,6 +463,7 @@
 
 
 def lin2lin(cp, size, size2):
+    _check_bytes(cp)
     _check_params(len(cp), size)
     _check_size(size2)
 
@@ -432,10 +480,14 @@
             sample <<= 24
         elif size == 2:
             sample <<= 16
+        elif size == 3:
+            sample <<= 8
         if size2 == 1:
             sample >>= 24
         elif size2 == 2:
             sample >>= 16
+        elif size2 == 3:
+            sample >>= 8
         sample = _overflow(sample, size2)
         _put_sample(result, size2, i, sample)
 
@@ -489,7 +541,8 @@
     nbytes = ceiling * bytes_per_frame
 
     rv = ffi.new("char[]", nbytes)
-    trim_index = lib.ratecv(rv, cp, frame_count, size,
+    cpbuf = ffi.from_buffer(cp)
+    trim_index = lib.ratecv(rv, cpbuf, frame_count, size,
                             nchannels, inrate, outrate,
                             state_d, prev_i, cur_i,
                             weightA, weightB)
@@ -505,6 +558,8 @@
             yield sample << 8
         elif size == 2:
             yield sample
+        elif size == 3:
+            yield sample >> 8
         elif size == 4:
             yield sample >> 16
 
@@ -514,6 +569,8 @@
         sample >>= 8
     elif size == 2:
         pass
+    elif size == 3:
+        sample <<= 8
     elif size == 4:
         sample <<= 16
     _put_sample(result, size, i, sample)
@@ -560,7 +617,8 @@
     state = _check_state(state)
     rv = ffi.new("unsigned char[]", len(cp) // size // 2)
     state_ptr = ffi.new("int[]", state)
-    lib.lin2adcpm(rv, cp, len(cp), size, state_ptr)
+    cpbuf = ffi.cast("unsigned char*", ffi.from_buffer(cp))
+    lib.lin2adcpm(rv, cpbuf, len(cp), size, state_ptr)
     return ffi.buffer(rv)[:], tuple(state_ptr)
 
 
@@ -569,7 +627,8 @@
     state = _check_state(state)
     rv = ffi.new("unsigned char[]", len(cp) * size * 2)
     state_ptr = ffi.new("int[]", state)
-    lib.adcpm2lin(rv, ffi.from_buffer(cp), len(cp), size, state_ptr)
+    cpbuf = ffi.cast("unsigned char*", ffi.from_buffer(cp))
+    lib.adcpm2lin(rv, cpbuf, len(cp), size, state_ptr)
     return ffi.buffer(rv)[:], tuple(state_ptr)
 
 def byteswap(cp, size):

From pypy.commits at gmail.com  Wed Feb 22 09:52:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 06:52:07 -0800 (PST)
Subject: [pypy-commit] pypy default: remove space.str0_w() completely
Message-ID: <58ada597.cfa8190a.e3d6a.231f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90299:f7ee914736ff
Date: 2017-02-22 15:40 +0100
http://bitbucket.org/pypy/pypy/changeset/f7ee914736ff/

Log:	remove space.str0_w() completely

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1622,10 +1622,6 @@
         """For tests only."""
         return self.bytes_w(w_obj)
 
-    @not_rpython    # tests only; should be replaced with bytes0_w or text0_w
-    def str0_w(self, w_obj):
-        return self.bytes0_w(w_obj)
-
     def bytes0_w(self, w_obj):
         "Like bytes_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -212,11 +212,11 @@
         res = self.space.interp_w(Function, w(None), can_be_None=True)
         assert res is None
 
-    def test_str0_w(self):
+    def test_text0_w(self):
         space = self.space
         w = space.wrap
-        assert space.str0_w(w("123")) == "123"
-        exc = space.raises_w(space.w_TypeError, space.str0_w, w("123\x004"))
+        assert space.text0_w(w("123")) == "123"
+        exc = space.raises_w(space.w_TypeError, space.text0_w, w("123\x004"))
         assert space.unicode0_w(w(u"123")) == u"123"
         exc = space.raises_w(space.w_TypeError, space.unicode0_w, w(u"123\x004"))
 

From pypy.commits at gmail.com  Wed Feb 22 09:52:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 06:52:02 -0800 (PST)
Subject: [pypy-commit] pypy default: Replace space.realstr_w ->
 space.realtext_w and 'str_or_None' ->
Message-ID: <58ada592.52152e0a.d48ff.d2b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90297:5b74b06f962b
Date: 2017-02-22 15:11 +0100
http://bitbucket.org/pypy/pypy/changeset/5b74b06f962b/

Log:	Replace space.realstr_w -> space.realtext_w and 'str_or_None' ->
	'text_or_none'. Fix a few places that accepted "str_or_None" but
	where it was really meant "fsencode or None".

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -420,7 +420,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         _args = arguments.from_object(space, w_args)
@@ -497,7 +497,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         bases_w = space.unpackiterable(w_bases)
@@ -1318,7 +1318,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.str_or_None_w(w_module)
+        _module = space.realtext_w(w_module) if not space.is_none(w_module) else space.w_None
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -1413,7 +1413,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         names_w = space.unpackiterable(w_names)
-        _names = [space.realstr_w(w_item) for w_item in names_w]
+        _names = [space.realtext_w(w_item) for w_item in names_w]
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
@@ -2495,7 +2495,7 @@
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
-        _attr = space.realstr_w(w_attr)
+        _attr = space.realtext_w(w_attr)
         if _attr is None:
             raise_required_value(space, w_node, 'attr')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -2592,7 +2592,7 @@
         w_ctx = get_field(space, w_node, 'ctx', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _id = space.realstr_w(w_id)
+        _id = space.realtext_w(w_id)
         if _id is None:
             raise_required_value(space, w_node, 'id')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -3415,8 +3415,8 @@
         w_defaults = get_field(space, w_node, 'defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [expr.from_object(space, w_item) for w_item in args_w]
-        _vararg = space.str_or_None_w(w_vararg)
-        _kwarg = space.str_or_None_w(w_kwarg)
+        _vararg = space.realtext_w(w_vararg) if not space.is_none(w_vararg) else space.w_None
+        _kwarg = space.realtext_w(w_kwarg) if not space.is_none(w_kwarg) else space.w_None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwarg, _defaults)
@@ -3448,7 +3448,7 @@
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', False)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.realstr_w(w_arg)
+        _arg = space.realtext_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
         _value = expr.from_object(space, w_value)
@@ -3482,10 +3482,10 @@
     def from_object(space, w_node):
         w_name = get_field(space, w_node, 'name', False)
         w_asname = get_field(space, w_node, 'asname', True)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.str_or_None_w(w_asname)
+        _asname = space.realtext_w(w_asname) if not space.is_none(w_asname) else space.w_None
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -150,8 +150,9 @@
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
-                return "space.str_or_None_w(%s)" % (value,)
-            return "space.realstr_w(%s)" % (value,)
+                return ("space.realtext_w(%s) if not space.is_none(%s) "
+                        "else space.w_None" % (value, value))
+            return "space.realtext_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
         elif field.type in ("bool",):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1604,9 +1604,9 @@
         else:
             return buf.as_str()
 
-    def str_or_None_w(self, w_obj):
-        # YYY rename
-        return None if self.is_none(w_obj) else self.bytes_w(w_obj)
+    def text_or_none_w(self, w_obj):
+        # return text_w(w_obj) or None
+        return None if self.is_none(w_obj) else self.text_w(w_obj)
 
     def bytes_w(self, w_obj):
         "Takes a bytes object and returns an unwrapped RPython bytestring."
@@ -1647,6 +1647,9 @@
                                      getfilesystemencoding(self))
         return self.bytes0_w(w_obj)
 
+    def fsencode_or_none_w(self, w_obj):
+        return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1681,9 +1684,9 @@
         """
         return w_obj.float_w(self, allow_conversion)
 
-    def realstr_w(self, w_obj):
-        # YYY rename
-        # Like bytes_w, but only works if w_obj is really of type 'str'.
+    def realtext_w(self, w_obj):
+        # Like bytes_w(), but only works if w_obj is really of type 'str'.
+        # On Python 3 this is the same as text_w().
         if not self.isinstance_w(w_obj, self.w_bytes):
             raise oefmt(self.w_TypeError, "argument must be a string")
         return self.bytes_w(w_obj)
@@ -1702,8 +1705,8 @@
         return rstring.assert_str0(result)
 
     def realunicode_w(self, w_obj):
-        # Like unicode_w, but only works if w_obj is really of type
-        # 'unicode'.
+        # Like unicode_w(), but only works if w_obj is really of type
+        # 'unicode'.  On Python 3 this is the same as unicode_w().
         if not self.isinstance_w(w_obj, self.w_unicode):
             raise oefmt(self.w_TypeError, "argument must be a unicode")
         return self.unicode_w(w_obj)
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -145,7 +145,7 @@
     def visit_bufferstr(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
-    def visit_str_or_None(self, el, app_sig):
+    def visit_text_or_none(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
     def visit_bytes(self, el, app_sig):
@@ -163,6 +163,9 @@
     def visit_fsencode(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_fsencode_or_none(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -286,8 +289,8 @@
     def visit_bufferstr(self, typ):
         self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),))
 
-    def visit_str_or_None(self, typ):
-        self.run_args.append("space.str_or_None_w(%s)" % (self.scopenext(),))
+    def visit_text_or_none(self, typ):
+        self.run_args.append("space.text_or_none_w(%s)" % (self.scopenext(),))
 
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
@@ -304,6 +307,9 @@
     def visit_fsencode(self, typ):
         self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
 
+    def visit_fsencode_or_none(self, typ):
+        self.run_args.append("space.fsencode_or_none_w(%s)" % (self.scopenext(),))
+
     def visit_nonnegint(self, typ):
         self.run_args.append("space.gateway_nonnegint_w(%s)" % (
             self.scopenext(),))
@@ -448,8 +454,8 @@
     def visit_bufferstr(self, typ):
         self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),))
 
-    def visit_str_or_None(self, typ):
-        self.unwrap.append("space.str_or_None_w(%s)" % (self.nextarg(),))
+    def visit_text_or_none(self, typ):
+        self.unwrap.append("space.text_or_none_w(%s)" % (self.nextarg(),))
 
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
@@ -466,6 +472,9 @@
     def visit_fsencode(self, typ):
         self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
 
+    def visit_fsencode_or_none(self, typ):
+        self.unwrap.append("space.fsencode_or_none_w(%s)" % (self.nextarg(),))
+
     def visit_nonnegint(self, typ):
         self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -25,7 +25,7 @@
 
 # helper
 
-def unpack_text_tuple(space,w_str_tuple):
+def unpack_text_tuple(space, w_str_tuple):
     return [space.text_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -377,7 +377,7 @@
             return space.wrap(s0+s1)
         app_g3_ss = gateway.interp2app_temp(g3_ss,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      'text', 'str_or_None'])
+                                                      'text', 'text_or_none'])
         w_app_g3_ss = space.wrap(app_g3_ss)
         assert self.space.eq_w(
             space.call(w_app_g3_ss,
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -572,7 +572,7 @@
         return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA)
 
 
-    @unwrap_spec(filename="str_or_None", flags=int)
+    @unwrap_spec(filename="fsencode_or_none", flags=int)
     def descr_dlopen(self, filename, flags=0):
         """\
 Load and return a dynamic library identified by 'name'.  The standard
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -91,7 +91,7 @@
 W_Library.typedef.acceptable_as_base_class = False
 
 
- at unwrap_spec(filename="str_or_None", flags=int)
+ at unwrap_spec(filename="fsencode_or_none", flags=int)
 def load_library(space, filename, flags=0):
     lib = W_Library(space, filename, flags)
     return lib
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -314,12 +314,12 @@
     w_res = space.call_function(w_encoder, w_obj, space.newtext(errors))
     return space.getitem(w_res, space.newint(0))
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def readbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('s#', w_data)
     return space.newtuple([space.newbytes(s), space.newint(len(s))])
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def charbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('t#', w_data)
     return space.newtuple([space.newbytes(s), space.newint(len(s))])
@@ -373,7 +373,7 @@
 def make_encoder_wrapper(name):
     rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
     assert hasattr(runicode, rname)
-    @unwrap_spec(uni=unicode, errors='str_or_None')
+    @unwrap_spec(uni=unicode, errors='text_or_none')
     def wrap_encoder(space, uni, errors="strict"):
         if errors is None:
             errors = 'strict'
@@ -387,7 +387,7 @@
 def make_decoder_wrapper(name):
     rname = "str_decode_%s" % (name.replace("_decode", ""), )
     assert hasattr(runicode, rname)
-    @unwrap_spec(string='bufferstr', errors='str_or_None',
+    @unwrap_spec(string='bufferstr', errors='text_or_none',
                  w_final=WrappedDefault(False))
     def wrap_decoder(space, string, errors="strict", w_final=None):
         if errors is None:
@@ -437,7 +437,7 @@
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=True"
- at unwrap_spec(uni=unicode, errors='str_or_None')
+ at unwrap_spec(uni=unicode, errors='text_or_none')
 def utf_8_encode(space, uni, errors="strict"):
     if errors is None:
         errors = 'strict'
@@ -450,7 +450,7 @@
         allow_surrogates=True)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
- at unwrap_spec(string='bufferstr', errors='str_or_None',
+ at unwrap_spec(string='bufferstr', errors='text_or_none',
              w_final = WrappedDefault(False))
 def utf_8_decode(space, string, errors="strict", w_final=None):
     if errors is None:
@@ -466,7 +466,7 @@
         allow_surrogates=True)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
+ at unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
     if errors is None:
@@ -487,7 +487,7 @@
     return space.newtuple([space.newunicode(res), space.newint(consumed),
                            space.newint(byteorder)])
 
- at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
+ at unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_32_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
     final = space.is_true(w_final)
@@ -585,7 +585,7 @@
             "character mapping must return integer, None or str")
 
 
- at unwrap_spec(string='bufferstr', errors='str_or_None')
+ at unwrap_spec(string='bufferstr', errors='text_or_none')
 def charmap_decode(space, string, errors="strict", w_mapping=None):
     if errors is None:
         errors = 'strict'
@@ -604,7 +604,7 @@
         final, state.decode_error_handler, mapping)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(uni=unicode, errors='str_or_None')
+ at unwrap_spec(uni=unicode, errors='text_or_none')
 def charmap_encode(space, uni, errors="strict", w_mapping=None):
     if errors is None:
         errors = 'strict'
@@ -647,7 +647,7 @@
             return -1
         return space.int_w(w_code)
 
- at unwrap_spec(string='bufferstr', errors='str_or_None',
+ at unwrap_spec(string='bufferstr', errors='text_or_none',
              w_final=WrappedDefault(False))
 def unicode_escape_decode(space, string, errors="strict", w_final=None):
     if errors is None:
@@ -667,7 +667,7 @@
 # ____________________________________________________________
 # Unicode-internal
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def unicode_internal_decode(space, w_string, errors="strict"):
     if errors is None:
         errors = 'strict'
@@ -691,7 +691,7 @@
 # support for the "string escape" codec
 # This is a bytes-to bytes transformation
 
- at unwrap_spec(data='bytes', errors='str_or_None')
+ at unwrap_spec(data='bytes', errors='text_or_none')
 def escape_encode(space, data, errors='strict'):
     from pypy.objspace.std.bytesobject import string_escape_encode
     result = string_escape_encode(data, quote="'")
@@ -701,7 +701,7 @@
     w_result = space.newbytes(result[start:end])
     return space.newtuple([w_result, space.newint(len(data))])
 
- at unwrap_spec(data='bytes', errors='str_or_None')
+ at unwrap_spec(data='bytes', errors='text_or_none')
 def escape_decode(space, data, errors='strict'):
     from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape
     result = PyString_DecodeEscape(space, data, errors, None)
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -666,7 +666,7 @@
     return False
 
 
- at unwrap_spec(w_file=W_File, encoding="str_or_None", errors="str_or_None")
+ at unwrap_spec(w_file=W_File, encoding="text_or_none", errors="text_or_none")
 def set_file_encoding(space, w_file, encoding=None, errors=None):
     w_file.encoding = encoding
     w_file.errors = errors
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -41,8 +41,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024
 
 @unwrap_spec(mode='text', buffering=int,
-             encoding="str_or_None", errors="str_or_None",
-             newline="str_or_None", closefd=bool)
+             encoding="text_or_none", errors="text_or_none",
+             newline="text_or_none", closefd=bool)
 def open(space, w_file, mode="r", buffering=-1, encoding=None, errors=None,
     newline=None, closefd=True):
     from pypy.module._io.interp_bufferedio import (W_BufferedRandom,
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -359,7 +359,7 @@
                                               # of the stream
         self.snapshot = None
 
-    @unwrap_spec(encoding="str_or_None", line_buffering=int)
+    @unwrap_spec(encoding="text_or_none", line_buffering=int)
     def descr_init(self, space, w_buffer, encoding=None,
                    w_errors=None, w_newline=None, line_buffering=0):
         self.state = STATE_ZERO
diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -68,7 +68,7 @@
         return space.newunicode(output)
 
 
- at unwrap_spec(errors="str_or_None")
+ at unwrap_spec(errors="text_or_none")
 def mbidecoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalDecoder, w_subtype)
     r.__init__(space, errors)
@@ -116,7 +116,7 @@
         return space.newbytes(output)
 
 
- at unwrap_spec(errors="str_or_None")
+ at unwrap_spec(errors="text_or_none")
 def mbiencoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalEncoder, w_subtype)
     r.__init__(space, errors)
diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py
--- a/pypy/module/_multibytecodec/interp_multibytecodec.py
+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py
@@ -11,7 +11,7 @@
         self.name = name
         self.codec = codec
 
-    @unwrap_spec(input='bytes', errors="str_or_None")
+    @unwrap_spec(input='bytes', errors="text_or_none")
     def decode(self, space, input, errors=None):
         if errors is None:
             errors = 'strict'
@@ -27,7 +27,7 @@
         return space.newtuple([space.newunicode(output),
                                space.newint(len(input))])
 
-    @unwrap_spec(input=unicode, errors="str_or_None")
+    @unwrap_spec(input=unicode, errors="text_or_none")
     def encode(self, space, input, errors=None):
         if errors is None:
             errors = 'strict'
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -344,7 +344,7 @@
     def getidentifier(self, space):
         return space.newint(self.cdll.getidentifier())
 
- at unwrap_spec(name='str_or_None', mode=int)
+ at unwrap_spec(name='fsencode_or_none', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
     return W_CDLL(space, name, mode)
 
@@ -363,7 +363,7 @@
         W_CDLL.__init__(self, space, name, mode)
         self.flags = libffi.FUNCFLAG_STDCALL
 
- at unwrap_spec(name='str_or_None', mode=int)
+ at unwrap_spec(name='fsencode_or_none', mode=int)
 def descr_new_windll(space, w_type, name, mode=-1):
     return W_WinDLL(space, name, mode)
 
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -235,7 +235,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(name='str_or_None')
+ at unwrap_spec(name='fsencode_or_none')
 def descr_new_cdll(space, w_type, name):
     cdll = open_cdll(space, name)
     return W_CDLL(space, name, cdll)
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -689,7 +689,7 @@
 
 The return value is the handle of the opened key.
 If the function fails, an EnvironmentError exception is raised."""
-    machine = space.str_or_None_w(w_machine)
+    machine = space.text_or_none_w(w_machine)
     hkey = hkey_w(w_hkey, space)
     with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
         ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -119,7 +119,7 @@
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     except ValueError:
-        if not space.isinstance_w(w_obj, space.w_str):
+        if not space.isinstance_w(w_obj, space.w_bytes):
             # XXX Python 3?
             raise BufferError("could not create buffer from object")
         view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.bytes_w(w_obj), track_allocation=False))
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -313,8 +313,8 @@
 
             return space.wrap(pydname)
 
-        @unwrap_spec(name='text', init='str_or_None', body='text',
-                     filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+        @unwrap_spec(name='text', init='text_or_none', body='text',
+                     filename='fsencode_or_none', PY_SSIZE_T_CLEAN=bool)
         def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -288,7 +288,7 @@
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_object = w_object
         self.w_start = w_start
@@ -628,11 +628,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.realstr_w(w_encoding)
-        space.realstr_w(w_object)
+        space.realtext_w(w_encoding)
+        space.realtext_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
@@ -718,11 +718,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.realstr_w(w_encoding)
+        space.realtext_w(w_encoding)
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -364,7 +364,7 @@
         length = space.len_w(w_fromlist)
         for i in range(length):
             w_name = space.getitem(w_fromlist, space.newint(i))
-            if not space.isinstance_w(w_name, space.w_str):
+            if not space.isinstance_w(w_name, space.w_text):
                 raise oefmt(space.w_TypeError,
                     "'fromlist' items must be str, not %T", w_name)
         if w_path is not None:
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -183,7 +183,7 @@
 
     def descr_setitem(self, w_index, w_value):
         space = self.space
-        value = space.realstr_w(w_value)
+        value = space.realtext_w(w_value)
         self.check_valid()
 
         self.check_writeable()
@@ -238,7 +238,7 @@
         if not space.isinstance_w(w_item, space.w_bytes):
             raise oefmt(space.w_IndexError,
                         "mmap slice assignment must be a string")
-        value = space.realstr_w(w_item)
+        value = space.realtext_w(w_item)
         if len(value) != (j - i):
             raise oefmt(space.w_IndexError,
                         "mmap slice assignment is wrong size")
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -181,7 +181,7 @@
         # we ignore w_type and always return a bytearray
         return new_bytearray(space, space.w_bytearray, data)
 
-    @unwrap_spec(encoding='str_or_None', errors='str_or_None')
+    @unwrap_spec(encoding='text_or_none', errors='text_or_none')
     def descr_init(self, space, w_source=None, encoding=None, errors=None):
         if w_source is None:
             w_source = space.newbytes('')

From pypy.commits at gmail.com  Wed Feb 22 09:52:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 06:52:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58ada594.85d5190a.f7948.21f3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90298:7921b4d69cc7
Date: 2017-02-22 15:37 +0100
http://bitbucket.org/pypy/pypy/changeset/7921b4d69cc7/

Log:	hg merge default

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1468,7 +1468,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.text_or_None_w(w_module)
+        _module = space.text_or_none_w(w_module)
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -3777,7 +3777,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         _type = expr.from_object(space, w_type)
-        _name = space.text_or_None_w(w_name)
+        _name = space.text_or_none_w(w_name)
         body_w = space.unpackiterable(w_body)
         _body = [stmt.from_object(space, w_item) for w_item in body_w]
         _lineno = space.int_w(w_lineno)
@@ -3946,7 +3946,7 @@
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', True)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.text_or_None_w(w_arg)
+        _arg = space.text_or_none_w(w_arg)
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
@@ -3981,7 +3981,7 @@
         _name = space.identifier_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.text_or_None_w(w_asname)
+        _asname = space.text_or_none_w(w_asname)
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -161,7 +161,7 @@
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
-                return "space.text_or_None_w(%s)" % (value,)
+                return "space.text_or_none_w(%s)" % (value,)
             return "space.identifier_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1580,8 +1580,7 @@
                 raise
         return self.buffer_w(w_obj, flags).as_str()
 
-
-    def text_or_None_w(self, w_obj):
+    def text_or_none_w(self, w_obj):
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
@@ -1630,6 +1629,9 @@
                         "argument must be a string without NUL characters")
         return rstring.assert_str0(result)
 
+    def fsencode_or_none_w(self, w_obj):
+        return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1677,13 +1679,6 @@
                         "characters")
         return rstring.assert_str0(result)
 
-    def realunicode_w(self, w_obj):
-        # Like unicode_w, but only works if w_obj is really of type
-        # 'unicode'.
-        if not self.isinstance_w(w_obj, self.w_unicode):
-            raise oefmt(self.w_TypeError, "argument must be a unicode")
-        return self.unicode_w(w_obj)
-
     def text_w(self, w_obj):
         """
         Unwrap a unicode object and return a 'utf-8-nosg' byte string
@@ -1692,6 +1687,9 @@
         """
         return w_obj.text_w(self)
 
+    realtext_w = text_w         # Python 2 compatibility
+    realunicode_w = unicode_w
+
     def identifier_w(self, w_obj):
         """
         Unwrap an object which is used as an identifier (i.e. names of
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -156,10 +156,7 @@
     def visit_bufferstr(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
-    def visit_str_or_None(self, el, app_sig):   # XXX kill me
-        self.checked_space_method(el, app_sig)
-
-    def visit_text_or_None(self, el, app_sig):
+    def visit_text_or_none(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
     def visit_bytes(self, el, app_sig):
@@ -177,6 +174,9 @@
     def visit_fsencode(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_fsencode_or_none(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -305,11 +305,8 @@
     def visit_bufferstr(self, typ):
         self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),))
 
-    def visit_str_or_None(self, typ):  #XXX kill me
-        self.run_args.append("space.text_or_None_w(%s)" % (self.scopenext(),))
-
-    def visit_text_or_None(self, typ):
-        self.run_args.append("space.text_or_None_w(%s)" % (self.scopenext(),))
+    def visit_text_or_none(self, typ):
+        self.run_args.append("space.text_or_none_w(%s)" % (self.scopenext(),))
 
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
@@ -326,6 +323,9 @@
     def visit_fsencode(self, typ):
         self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
 
+    def visit_fsencode_or_none(self, typ):
+        self.run_args.append("space.fsencode_or_none_w(%s)" % (self.scopenext(),))
+
     def visit_nonnegint(self, typ):
         self.run_args.append("space.gateway_nonnegint_w(%s)" % (
             self.scopenext(),))
@@ -473,11 +473,8 @@
     def visit_bufferstr(self, typ):
         self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),))
 
-    def visit_str_or_None(self, typ):  #XXX kill me
-        self.unwrap.append("space.text_or_None_w(%s)" % (self.nextarg(),))
-
-    def visit_text_or_None(self, typ):
-        self.unwrap.append("space.text_or_None_w(%s)" % (self.nextarg(),))
+    def visit_text_or_none(self, typ):
+        self.unwrap.append("space.text_or_none_w(%s)" % (self.nextarg(),))
 
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
@@ -494,6 +491,9 @@
     def visit_fsencode(self, typ):
         self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
 
+    def visit_fsencode_or_none(self, typ):
+        self.unwrap.append("space.fsencode_or_none_w(%s)" % (self.nextarg(),))
+
     def visit_nonnegint(self, typ):
         self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -25,7 +25,7 @@
 
 # helper
 
-def unpack_text_tuple(space,w_str_tuple):
+def unpack_text_tuple(space, w_str_tuple):
     return [space.text_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -384,7 +384,7 @@
             return space.wrap(s0+s1)
         app_g3_ss = gateway.interp2app_temp(g3_ss,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      'text', 'str_or_None'])
+                                                      'text', 'text_or_none'])
         w_app_g3_ss = space.wrap(app_g3_ss)
         assert self.space.eq_w(
             space.call(w_app_g3_ss,
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -572,7 +572,7 @@
         return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA)
 
 
-    @unwrap_spec(filename="str_or_None", flags=int)
+    @unwrap_spec(filename="fsencode_or_none", flags=int)
     def descr_dlopen(self, filename, flags=0):
         """\
 Load and return a dynamic library identified by 'name'.  The standard
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -91,7 +91,7 @@
 W_Library.typedef.acceptable_as_base_class = False
 
 
- at unwrap_spec(filename="str_or_None", flags=int)
+ at unwrap_spec(filename="fsencode_or_none", flags=int)
 def load_library(space, filename, flags=0):
     lib = W_Library(space, filename, flags)
     return lib
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -539,7 +539,7 @@
     w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
     return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def readbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('s#', w_data)
     return space.newtuple([space.newbytes(s), space.newint(len(s))])
@@ -618,7 +618,7 @@
 def make_encoder_wrapper(name):
     rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
     assert hasattr(runicode, rname)
-    @unwrap_spec(uni=unicode, errors='str_or_None')
+    @unwrap_spec(uni=unicode, errors='text_or_none')
     def wrap_encoder(space, uni, errors="strict"):
         if errors is None:
             errors = 'strict'
@@ -632,7 +632,7 @@
 def make_decoder_wrapper(name):
     rname = "str_decode_%s" % (name.replace("_decode", ""), )
     assert hasattr(runicode, rname)
-    @unwrap_spec(string='bufferstr', errors='str_or_None',
+    @unwrap_spec(string='bufferstr', errors='text_or_none',
                  w_final=WrappedDefault(False))
     def wrap_decoder(space, string, errors="strict", w_final=None):
         if errors is None:
@@ -677,7 +677,7 @@
 if hasattr(runicode, 'str_decode_mbcs'):
     # mbcs functions are not regular, because we have to pass
     # "force_ignore/replace=False"
-    @unwrap_spec(uni=unicode, errors='str_or_None')
+    @unwrap_spec(uni=unicode, errors='text_or_none')
     def mbcs_encode(space, uni, errors="strict"):
         if errors is None:
             errors = 'strict'
@@ -687,7 +687,7 @@
             force_replace=False)
         return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
-    @unwrap_spec(string='bufferstr', errors='str_or_None',
+    @unwrap_spec(string='bufferstr', errors='text_or_none',
                  w_final=WrappedDefault(False))
     def mbcs_decode(space, string, errors="strict", w_final=None):
         if errors is None:
@@ -702,7 +702,7 @@
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=False"
- at unwrap_spec(uni=unicode, errors='str_or_None')
+ at unwrap_spec(uni=unicode, errors='text_or_none')
 def utf_8_encode(space, uni, errors="strict"):
     if errors is None:
         errors = 'strict'
@@ -715,7 +715,7 @@
         allow_surrogates=False)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
- at unwrap_spec(string='bufferstr', errors='str_or_None',
+ at unwrap_spec(string='bufferstr', errors='text_or_none',
              w_final = WrappedDefault(False))
 def utf_8_decode(space, string, errors="strict", w_final=None):
     if errors is None:
@@ -731,7 +731,7 @@
         allow_surrogates=False)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
+ at unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
     if errors is None:
@@ -752,7 +752,7 @@
     return space.newtuple([space.newunicode(res), space.newint(consumed),
                            space.newint(byteorder)])
 
- at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
+ at unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_32_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
     final = space.is_true(w_final)
@@ -850,7 +850,7 @@
             "character mapping must return integer, bytes or None, not str")
 
 
- at unwrap_spec(string='bufferstr', errors='str_or_None')
+ at unwrap_spec(string='bufferstr', errors='text_or_none')
 def charmap_decode(space, string, errors="strict", w_mapping=None):
     if errors is None:
         errors = 'strict'
@@ -869,7 +869,7 @@
         final, state.decode_error_handler, mapping)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(uni=unicode, errors='str_or_None')
+ at unwrap_spec(uni=unicode, errors='text_or_none')
 def charmap_encode(space, uni, errors="strict", w_mapping=None):
     if errors is None:
         errors = 'strict'
@@ -912,7 +912,7 @@
             return -1
         return space.int_w(w_code)
 
- at unwrap_spec(errors='str_or_None', w_final=WrappedDefault(False))
+ at unwrap_spec(errors='text_or_none', w_final=WrappedDefault(False))
 def unicode_escape_decode(space, w_string, errors="strict", w_final=None):
     string = space.getarg_w('s*', w_string).as_str()
     if errors is None:
@@ -932,7 +932,7 @@
 # ____________________________________________________________
 # Raw Unicode escape (accepts bytes or str)
 
- at unwrap_spec(errors='str_or_None', w_final=WrappedDefault(False))
+ at unwrap_spec(errors='text_or_none', w_final=WrappedDefault(False))
 def raw_unicode_escape_decode(space, w_string, errors="strict", w_final=None):
     string = space.getarg_w('s*', w_string).as_str()
     if errors is None:
@@ -947,7 +947,7 @@
 # ____________________________________________________________
 # Unicode-internal
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def unicode_internal_decode(space, w_string, errors="strict"):
     if errors is None:
         errors = 'strict'
@@ -969,7 +969,7 @@
         final, state.decode_error_handler)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def unicode_internal_encode(space, w_uni, errors="strict"):
     space.warn(space.newtext("unicode_internal codec has been deprecated"),
                space.w_DeprecationWarning)
@@ -990,13 +990,13 @@
 # support for the "string escape" translation
 # This is a bytes-to bytes transformation
 
- at unwrap_spec(data='bytes', errors='str_or_None')
+ at unwrap_spec(data='bytes', errors='text_or_none')
 def escape_encode(space, data, errors='strict'):
     from pypy.objspace.std.bytesobject import string_escape_encode
     result = string_escape_encode(data, False)
     return space.newtuple([space.newbytes(result), space.newint(len(data))])
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def escape_decode(space, w_data, errors='strict'):
     data = space.getarg_w('s#', w_data)
     from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -15,8 +15,8 @@
             space.newtuple([space.w_ValueError, space.w_IOError]))
 
 @unwrap_spec(mode='text', buffering=int,
-             encoding="str_or_None", errors="str_or_None",
-             newline="str_or_None", closefd=int)
+             encoding="text_or_none", errors="text_or_none",
+             newline="text_or_none", closefd=int)
 def open(space, w_file, mode="r", buffering=-1, encoding=None, errors=None,
          newline=None, closefd=True, w_opener=None):
     from pypy.module._io.interp_bufferedio import (W_BufferedRandom,
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -374,7 +374,7 @@
                                               # of the stream
         self.snapshot = None
 
-    @unwrap_spec(encoding="str_or_None", line_buffering=int, write_through=int)
+    @unwrap_spec(encoding="text_or_none", line_buffering=int, write_through=int)
     def descr_init(self, space, w_buffer, encoding=None,
                    w_errors=None, w_newline=None, line_buffering=0,
                    write_through=0):
diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -69,7 +69,7 @@
         return space.newunicode(output)
 
 
- at unwrap_spec(errors="str_or_None")
+ at unwrap_spec(errors="text_or_none")
 def mbidecoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalDecoder, w_subtype)
     r.__init__(space, errors)
@@ -118,7 +118,7 @@
         return space.newbytes(output)
 
 
- at unwrap_spec(errors="str_or_None")
+ at unwrap_spec(errors="text_or_none")
 def mbiencoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalEncoder, w_subtype)
     r.__init__(space, errors)
diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py
--- a/pypy/module/_multibytecodec/interp_multibytecodec.py
+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py
@@ -11,7 +11,7 @@
         self.name = name
         self.codec = codec
 
-    @unwrap_spec(input='bufferstr', errors="str_or_None")
+    @unwrap_spec(input='bufferstr', errors="text_or_none")
     def decode(self, space, input, errors=None):
         if errors is None:
             errors = 'strict'
@@ -27,7 +27,7 @@
         return space.newtuple([space.newunicode(output),
                                space.newint(len(input))])
 
-    @unwrap_spec(input=unicode, errors="str_or_None")
+    @unwrap_spec(input=unicode, errors="text_or_none")
     def encode(self, space, input, errors=None):
         if errors is None:
             errors = 'strict'
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -525,7 +525,7 @@
     def after_fork(self):
         self.count = 0
 
-    @unwrap_spec(kind=int, maxvalue=int, name='text_or_None')
+    @unwrap_spec(kind=int, maxvalue=int, name='text_or_none')
     def rebuild(space, w_cls, w_handle, kind, maxvalue, name):
         #
         if sys_platform != 'win32' and name is not None:
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -344,7 +344,7 @@
     def getidentifier(self, space):
         return space.newint(self.cdll.getidentifier())
 
- at unwrap_spec(name='str_or_None', mode=int)
+ at unwrap_spec(name='fsencode_or_none', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
     return W_CDLL(space, name, mode)
 
@@ -363,7 +363,7 @@
         W_CDLL.__init__(self, space, name, mode)
         self.flags = libffi.FUNCFLAG_STDCALL
 
- at unwrap_spec(name='str_or_None', mode=int)
+ at unwrap_spec(name='fsencode_or_none', mode=int)
 def descr_new_windll(space, w_type, name, mode=-1):
     return W_WinDLL(space, name, mode)
 
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -242,7 +242,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(name='str_or_None')
+ at unwrap_spec(name='fsencode_or_none')
 def descr_new_cdll(space, w_type, name):
     cdll = open_cdll(space, name)
     return W_CDLL(space, name, cdll)
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -688,7 +688,7 @@
 
 The return value is the handle of the opened key.
 If the function fails, an EnvironmentError exception is raised."""
-    machine = space.str_or_None_w(w_machine)
+    machine = space.text_or_none_w(w_machine)
     hkey = hkey_w(w_hkey, space)
     with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
         ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -315,8 +315,8 @@
 
             return space.wrap(pydname)
 
-        @unwrap_spec(name='text', init='str_or_None', body='text',
-                     filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+        @unwrap_spec(name='text', init='text_or_none', body='text',
+                     filename='fsencode_or_none', PY_SSIZE_T_CLEAN=bool)
         def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -358,7 +358,7 @@
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.text_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_object = w_object
         self.w_start = w_start
@@ -908,11 +908,11 @@
             w_bytes = space.newbytes(space.bufferstr_w(w_object))
         else:
             w_bytes = w_object
-        space.text_w(w_encoding)
+        space.realtext_w(w_encoding)
         space.bytes_w(w_bytes)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.text_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_bytes
@@ -1001,11 +1001,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.text_w(w_encoding)
+        space.realtext_w(w_encoding)
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.text_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -206,7 +206,7 @@
         # we ignore w_type and always return a bytearray
         return new_bytearray(space, space.w_bytearray, data)
 
-    @unwrap_spec(encoding='text_or_None', errors='text_or_None')
+    @unwrap_spec(encoding='text_or_none', errors='text_or_none')
     def descr_init(self, space, w_source=None, encoding=None, errors=None):
         assert isinstance(self, W_BytearrayObject)
         data = [c for c in newbytesdata_w(space, w_source, encoding, errors)]
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -526,7 +526,7 @@
         return space.newlist_bytes(lst)
 
     @staticmethod
-    @unwrap_spec(encoding='text_or_None', errors='text_or_None')
+    @unwrap_spec(encoding='text_or_none', errors='text_or_none')
     def descr_new(space, w_stringtype, w_source=None, encoding=None,
                   errors=None):
         if (w_source and space.is_w(w_stringtype, space.w_bytes)

From pypy.commits at gmail.com  Wed Feb 22 09:52:58 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Feb 2017 06:52:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: obscure,
 python 3.5.2 has a different message than 3.5.3
Message-ID: <58ada5ca.52152e0a.d48ff.d2be@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90300:4d3783a2d4a2
Date: 2017-02-22 15:50 +0100
http://bitbucket.org/pypy/pypy/changeset/4d3783a2d4a2/

Log:	obscure, python 3.5.2 has a different message than 3.5.3

diff --git a/lib-python/3/test/test_builtin.py b/lib-python/3/test/test_builtin.py
--- a/lib-python/3/test/test_builtin.py
+++ b/lib-python/3/test/test_builtin.py
@@ -1475,6 +1475,8 @@
             with self.assertRaisesRegex(TypeError,
                                         r'\b%s\b' % re.escape(cls.__name__)):
                 format(obj, 's')
+
+
         # --------------------------------------------------------------------
 
         # make sure we can take a subclass of str as a format spec
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -221,7 +221,8 @@
         raise oefmt(space.w_TypeError, "format_spec must be a string")
     if space.len_w(w_format_spec) > 0:
         raise oefmt(space.w_TypeError,
-                    "non-empty format string passed to object.__format__")
+                     "unsupported format string passed to %T.__format__",
+                     w_obj);
     return space.format(w_as_str, w_format_spec)
 
 def descr__eq__(space, w_self, w_other):
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -97,6 +97,16 @@
         assert res == "Pickle"
         assert isinstance(res, str)
 
+    def test_format(self):
+        class B:
+            pass
+        try:
+            format(B(), 's')
+            assert False, "must not pass the previous call"
+        except TypeError as e:
+            assert 'B.__format__' in str(e)
+
+
     def test_subclasshook(self):
         class x(object):
             pass

From pypy.commits at gmail.com  Wed Feb 22 09:55:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 06:55:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: revert this part of 4d3783a2d4a2 (avoid
 introducing pointless difference)
Message-ID: <58ada676.1e142e0a.57aef.2269@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90301:2cd836c625bf
Date: 2017-02-22 15:55 +0100
http://bitbucket.org/pypy/pypy/changeset/2cd836c625bf/

Log:	revert this part of 4d3783a2d4a2 (avoid introducing pointless
	difference)

diff --git a/lib-python/3/test/test_builtin.py b/lib-python/3/test/test_builtin.py
--- a/lib-python/3/test/test_builtin.py
+++ b/lib-python/3/test/test_builtin.py
@@ -1475,8 +1475,6 @@
             with self.assertRaisesRegex(TypeError,
                                         r'\b%s\b' % re.escape(cls.__name__)):
                 format(obj, 's')
-
-
         # --------------------------------------------------------------------
 
         # make sure we can take a subclass of str as a format spec

From pypy.commits at gmail.com  Wed Feb 22 10:11:46 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 22 Feb 2017 07:11:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use raises() instead of a hand-rolled
 equivalent
Message-ID: <58adaa32.07582e0a.99b4a.cf67@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90302:c13e668a628c
Date: 2017-02-22 15:11 +0000
http://bitbucket.org/pypy/pypy/changeset/c13e668a628c/

Log:	Use raises() instead of a hand-rolled equivalent

diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -100,11 +100,8 @@
     def test_format(self):
         class B:
             pass
-        try:
-            format(B(), 's')
-            assert False, "must not pass the previous call"
-        except TypeError as e:
-            assert 'B.__format__' in str(e)
+        excinfo = raises(TypeError, format, B(), 's')
+        assert 'B.__format__' in str(excinfo.value)
 
 
     def test_subclasshook(self):

From pypy.commits at gmail.com  Wed Feb 22 10:12:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 07:12:55 -0800 (PST)
Subject: [pypy-commit] pypy default: remove 'space.w_str'
Message-ID: <58adaa77.0d012e0a.673d0.e10c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90303:f325422fa569
Date: 2017-02-22 16:12 +0100
http://bitbucket.org/pypy/pypy/changeset/f325422fa569/

Log:	remove 'space.w_str'

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -45,7 +45,6 @@
 
     def str_w(self, space):
         return NonConstant("foobar")
-    identifier_w = bytes_w = str_w
 
     def unicode_w(self, space):
         return NonConstant(u"foobar")
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -89,12 +89,15 @@
         for typedef, cls in builtin_type_classes.items():
             w_type = self.gettypeobject(typedef)
             self.builtin_types[typedef.name] = w_type
-            if 1: # typedef.name != "str":      BACKCOMPAT
-                setattr(self, 'w_' + typedef.name, w_type)
-            if typedef.name == "str":
-                self.w_bytes = w_type
+            name = typedef.name
+            # we don't expose 'space.w_str' at all, to avoid confusion
+            # with Python 3.  Instead, in Python 2, it becomes
+            # space.w_bytes (or space.w_text).
+            if name == 'str':
+                name = 'bytes'
+            setattr(self, 'w_' + name, w_type)
             self._interplevel_classes[w_type] = cls
-        self.w_text = self.w_bytes # this is w_unicode on Py3
+        self.w_text = self.w_bytes   # 'space.w_text' is w_unicode on Py3
         self.w_dict.flag_map_or_seq = 'M'
         self.builtin_types["NotImplemented"] = self.w_NotImplemented
         self.builtin_types["Ellipsis"] = self.w_Ellipsis

From pypy.commits at gmail.com  Wed Feb 22 10:15:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 07:15:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Rename 'space.wrap_fsdecoded' ->
 'space.newfilename'
Message-ID: <58adab13.ca052e0a.2b677.caa3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90304:0deea03316b3
Date: 2017-02-22 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/0deea03316b3/

Log:	Rename 'space.wrap_fsdecoded' -> 'space.newfilename'

diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -145,7 +145,7 @@
         command = args.pop(0)
     for arg in args:
         space.call_method(space.sys.get('argv'), 'append',
-                          space.wrap_fsdecoded(arg))
+                          space.newfilename(arg))
 
     # load the source of the program given as command-line argument
     if interactiveconfig.runcommand:
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -71,8 +71,8 @@
                         rlocale.setlocale(rlocale.LC_CTYPE, '')
                     except rlocale.LocaleError:
                         pass
-                w_executable = space.wrap_fsdecoded(argv[0])
-                w_argv = space.newlist([space.wrap_fsdecoded(s)
+                w_executable = space.newfilename(argv[0])
+                w_argv = space.newlist([space.newfilename(s)
                                         for s in argv[1:]])
                 w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
                 exitcode = space.int_w(w_exitcode)
@@ -130,7 +130,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap_fsdecoded(home), w_initstdio],
+            space.appexec([w_path, space.newfilename(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -20,7 +20,7 @@
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
     w_msg = space.newtext(msg)
-    w_filename = space.wrap_fsdecoded(fn)
+    w_filename = space.newfilename(fn)
     w_lineno = space.newint(lineno)
     w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1725,10 +1725,6 @@
             w_obj = self.fsdecode(w_obj)
         return self.unicode0_w(w_obj)
 
-    # BACKCOMPAT  -- replace me with newfilename()
-    def wrap_fsdecoded(self, x):
-        return self.newfilename(x)
-
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
         # For cases where you need to accept bools and ints and nothing
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -663,9 +663,9 @@
     w_filename = None
     w_filename2 = None
     if filename is not None:
-        w_filename = space.wrap_fsdecoded(filename)
+        w_filename = space.newfilename(filename)
         if filename2 is not None:
-            w_filename2 = space.wrap_fsdecoded(filename2)
+            w_filename2 = space.newfilename(filename2)
     return wrap_oserror2(space, e, w_filename,
                          exception_name=exception_name,
                          w_exception_class=w_exception_class,
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -43,7 +43,7 @@
         space.setitem(w_globals, space.newtext('__builtins__'), space.builtin)
         if filename is not None:
             space.setitem(w_globals, space.newtext('__file__'),
-                          space.wrap_fsdecoded(filename))
+                          space.newfilename(filename))
 
         retval = pycode.exec_code(space, w_globals, w_globals)
         if eval:
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -30,7 +30,7 @@
                                            'replace')
             w_text = space.newunicode(text)
         if self.filename is not None:
-            w_filename = space.wrap_fsdecoded(self.filename)
+            w_filename = space.newfilename(self.filename)
         return space.newtuple([space.newtext(self.msg),
                                space.newtuple([w_filename,
                                                space.newint(self.lineno),
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -120,7 +120,7 @@
         w_EnvironmentError = [EnvironmentError]
         def wrap(self, obj):
             return [obj]
-        newint = newtext = newunicode = wrap_fsdecoded = wrap
+        newint = newtext = newunicode = newfilename = wrap
         def call_function(self, exc, w_errno, w_msg, w_filename=None):
             return (exc, w_errno, w_msg, w_filename)
     space = FakeSpace()
diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
--- a/pypy/interpreter/test/test_fsencode.py
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -77,7 +77,7 @@
             assert space.fsdecode_w(w_enc) == st
 
             assert space.fsencode_w(w_enc) == space.bytes_w(w_enc)
-            assert space.eq_w(space.wrap_fsdecoded(space.bytes_w(w_enc)), w_st2)
+            assert space.eq_w(space.newfilename(space.bytes_w(w_enc)), w_st2)
 
     def test_null_byte(self):
         space = self.space
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -615,7 +615,7 @@
     co_varnames = GetSetProperty(PyCode.fget_co_varnames),
     co_freevars = GetSetProperty(PyCode.fget_co_freevars),
     co_cellvars = GetSetProperty(PyCode.fget_co_cellvars),
-    co_filename = interp_attrproperty('co_filename', cls=PyCode, wrapfn="wrap_fsdecoded"),
+    co_filename = interp_attrproperty('co_filename', cls=PyCode, wrapfn="newfilename"),
     co_name = interp_attrproperty('co_name', cls=PyCode, wrapfn="newtext"),
     co_firstlineno = interp_attrproperty('co_firstlineno', cls=PyCode, wrapfn="newint"),
     co_lnotab = interp_attrproperty('co_lnotab', cls=PyCode, wrapfn="newbytes"),
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -42,7 +42,7 @@
     w_name = space.newtext(name)
     module = Module(space, w_name)
     if path is not None:
-        module.setdictvalue(space, '__file__', space.wrap_fsdecoded(path))
+        module.setdictvalue(space, '__file__', space.newfilename(path))
     module.setdictvalue(space, 'ffi', ffi)
     module.setdictvalue(space, 'lib', lib)
     w_modules_dict = space.sys.get('modules')
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -40,7 +40,7 @@
             # Linux abstract namespace
             return space.newbytes(path)
         else:
-            return space.wrap_fsdecoded(path)
+            return space.newfilename(path)
     elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress):
         return space.newtuple([space.newint(addr.get_pid()),
                                space.newint(addr.get_groups())])
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1495,9 +1495,9 @@
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
         w_name = space.newunicode(name.decode('ascii'))
-        w_path = space.wrap_fsdecoded(path)
+        w_path = space.newfilename(path)
         raise raise_import_error(space,
-            space.wrap_fsdecoded(e.msg), w_name, w_path)
+            space.newfilename(e.msg), w_name, w_path)
     look_for = None
     #
     if space.config.objspace.usemodules._cffi_backend:
@@ -1528,9 +1528,9 @@
         else:
             look_for = also_look_for
     msg = u"function %s not found in library %s" % (
-        unicode(look_for), space.unicode_w(space.wrap_fsdecoded(path)))
+        unicode(look_for), space.unicode_w(space.newfilename(path)))
     w_name = space.newunicode(name.decode('ascii'))
-    w_path = space.wrap_fsdecoded(path)
+    w_path = space.newfilename(path)
     raise_import_error(space, space.newunicode(msg), w_name, w_path)
 
 
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -145,7 +145,7 @@
     else:
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
-    space.setattr(w_mod, space.newtext('__file__'), space.wrap_fsdecoded(pathname))
+    space.setattr(w_mod, space.newtext('__file__'), space.newfilename(pathname))
     cpathname = importing.make_compiled_pathname(pathname)
     importing.exec_code_module(space, w_mod, code, pathname, cpathname)
     return w_mod
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -47,7 +47,7 @@
     state.package_context = None, None
 
     if f_path is not None:
-        dict_w = {'__file__': space.wrap_fsdecoded(f_path)}
+        dict_w = {'__file__': space.newfilename(f_path)}
     else:
         dict_w = {}
     convert_method_defs(space, dict_w, methods, None, w_mod, modname)
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -147,7 +147,7 @@
     # XXX Doesn't actually do anything with PyErr_CheckSignals.
     if llfilename:
         filename = rffi.charp2str(llfilename)
-        w_filename = space.wrap_fsdecoded(filename)
+        w_filename = space.newfilename(filename)
     else:
         w_filename = space.w_None
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -256,9 +256,9 @@
         if pathname is not None:
             w_pathname = get_sourcefile(space, pathname)
         else:
-            w_pathname = space.wrap_fsdecoded(code_w.co_filename)
+            w_pathname = space.newfilename(code_w.co_filename)
         if cpathname is not None:
-            w_cpathname = space.wrap_fsdecoded(cpathname)
+            w_cpathname = space.newfilename(cpathname)
         else:
             w_cpathname = space.w_None
         space.setitem(w_dict, space.newtext("__file__"), w_pathname)
@@ -329,7 +329,7 @@
     start = len(filename) - 4
     stop = len(filename) - 1
     if not 0 <= start <= stop or filename[start:stop].lower() != ".py":
-        return space.wrap_fsdecoded(filename)
+        return space.newfilename(filename)
     py = make_source_pathname(filename)
     if py is None:
         py = filename[:-1]
@@ -339,8 +339,8 @@
         pass
     else:
         if stat.S_ISREG(st.st_mode):
-            return space.wrap_fsdecoded(py)
-    return space.wrap_fsdecoded(filename)
+            return space.newfilename(py)
+    return space.newfilename(filename)
 
 def update_code_filenames(space, code_w, pathname, oldname=None):
     assert isinstance(code_w, PyCode)
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -117,7 +117,7 @@
 
     if state is not None:    # 'None' for testing only
         lib_extensions = os.path.join(lib_pypy, '__extensions__')
-        state.w_lib_extensions = state.space.wrap_fsdecoded(lib_extensions)
+        state.w_lib_extensions = state.space.newfilename(lib_extensions)
         importlist.append(lib_extensions)
 
     importlist.append(lib_pypy)
@@ -149,12 +149,12 @@
 
 @unwrap_spec(executable='fsencode')
 def pypy_find_executable(space, executable):
-    return space.wrap_fsdecoded(find_executable(executable))
+    return space.newfilename(find_executable(executable))
 
 
 @unwrap_spec(filename='fsencode')
 def pypy_resolvedirof(space, filename):
-    return space.wrap_fsdecoded(resolvedirof(filename))
+    return space.newfilename(resolvedirof(filename))
 
 
 @unwrap_spec(executable='fsencode')
@@ -171,12 +171,12 @@
                 path, prefix = find_stdlib(get_state(space), dyn_path)
         if path is None:
             return space.w_None
-    w_prefix = space.wrap_fsdecoded(prefix)
+    w_prefix = space.newfilename(prefix)
     space.setitem(space.sys.w_dict, space.newtext('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.newtext('exec_prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.newtext('base_prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.newtext('base_exec_prefix'), w_prefix)
-    return space.newlist([space.wrap_fsdecoded(p) for p in path])
+    return space.newlist([space.newfilename(p) for p in path])
 
 def pypy_initfsencoding(space):
     space.sys.filesystemencoding = _getfilesystemencoding(space)
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -22,7 +22,7 @@
         # Initialize the default path
         srcdir = os.path.dirname(pypydir)
         path = compute_stdlib_path(self, srcdir)
-        self.w_path = space.newlist([space.wrap_fsdecoded(p) for p in path])
+        self.w_path = space.newlist([space.newfilename(p) for p in path])
 
 def get(space):
     return space.fromcache(State)
@@ -31,4 +31,4 @@
     """NOT_RPYTHON
     (should be removed from interpleveldefs before translation)"""
     from rpython.tool.udir import udir
-    return space.wrap_fsdecoded(str(udir))
+    return space.newfilename(str(udir))
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -53,9 +53,9 @@
         try:
             w_zipimporter = self.cache[name]
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap_fsdecoded(name))
+            raise OperationError(space.w_KeyError, space.newfilename(name))
         assert isinstance(w_zipimporter, W_ZipImporter)
-        w_fs = space.wrap_fsdecoded
+        w_fs = space.newfilename
         w_d = space.newdict()
         for key, info in w_zipimporter.zip_file.NameToInfo.iteritems():
             if ZIPSEP != os.path.sep:
@@ -67,7 +67,7 @@
         return w_d
 
     def keys(self, space):
-        return space.newlist([space.wrap_fsdecoded(s)
+        return space.newlist([space.newfilename(s)
                               for s in self.cache.keys()])
 
     def values(self, space):
@@ -76,7 +76,7 @@
         return space.newlist(values_w)
 
     def items(self, space):
-        w_fs = space.wrap_fsdecoded
+        w_fs = space.newfilename
         items_w = [space.newtuple([w_fs(key), self._getitem(space, key)])
                    for key in self.cache.keys()]
         return space.newlist(items_w)
@@ -128,8 +128,8 @@
 
     def getprefix(self, space):
         if ZIPSEP == os.path.sep:
-            return space.wrap_fsdecoded(self.prefix)
-        return space.wrap_fsdecoded(self.prefix.replace(ZIPSEP, os.path.sep))
+            return space.newfilename(self.prefix)
+        return space.newfilename(self.prefix.replace(ZIPSEP, os.path.sep))
 
     def _find_relative_path(self, filename):
         if filename.startswith(self.filename):
@@ -147,7 +147,7 @@
             return fname
 
     def import_py_file(self, space, modname, filename, buf, pkgpath):
-        w_mod = Module(space, space.wrap_fsdecoded(modname))
+        w_mod = Module(space, space.newfilename(modname))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
         space.setattr(w_mod, space.newtext('__loader__'), self)
         importing._prepare_module(space, w_mod, real_name, pkgpath)
@@ -316,7 +316,7 @@
                 return w_code
         raise oefmt(get_error(space),
                     "Cannot find source or code for %R in %R",
-                    w_fullname, space.wrap_fsdecoded(self.name))
+                    w_fullname, space.newfilename(self.name))
 
     @unwrap_spec(fullname='fsencode')
     def get_source(self, space, fullname):
@@ -337,20 +337,20 @@
             return space.w_None
         raise oefmt(get_error(space),
                     "Cannot find source for %R in %R",
-                    space.wrap_fsdecoded(filename),
-                    space.wrap_fsdecoded(self.name))
+                    space.newfilename(filename),
+                    space.newfilename(self.name))
 
     def get_filename(self, space, w_fullname):
         fullname = space.fsencode_w(w_fullname)
         filename = self.make_filename(fullname)
         for _, is_package, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
-                return space.wrap_fsdecoded(self.filename + os.path.sep +
+                return space.newfilename(self.filename + os.path.sep +
                                             self.corr_zname(filename + ext))
         raise oefmt(get_error(space),
                     "Cannot find module %R in %R",
-                    space.wrap_fsdecoded(filename),
-                    space.wrap_fsdecoded(self.name))
+                    space.newfilename(filename),
+                    space.newfilename(self.name))
 
     def is_package(self, space, w_fullname):
         fullname = space.fsencode_w(w_fullname)
@@ -360,12 +360,12 @@
                 return space.newbool(is_package)
         raise oefmt(get_error(space),
                     "Cannot find module %R in %R",
-                    space.wrap_fsdecoded(filename),
-                    space.wrap_fsdecoded(self.name))
+                    space.newfilename(filename),
+                    space.newfilename(self.name))
 
     def getarchive(self, space):
         space = self.space
-        return space.wrap_fsdecoded(self.filename)
+        return space.newfilename(self.filename)
 
     def _find_loader(self, space, fullname):
         filename = self.make_filename(fullname)
@@ -387,7 +387,7 @@
             result = [self, space.newlist([])]
         else:
             result = [space.w_None,
-                      space.newlist([space.wrap_fsdecoded(ns_portion)])]
+                      space.newlist([space.newfilename(ns_portion)])]
         return space.newtuple(result)
 
 def descr_new_zipimporter(space, w_type, w_name):
@@ -423,7 +423,7 @@
         zip_file = RZipFile(filename, 'r')
     except (BadZipfile, OSError):
         raise oefmt(get_error(space), "%R seems not to be a zipfile",
-                    space.wrap_fsdecoded(filename))
+                    space.newfilename(filename))
     except RZlibError as e:
         # in this case, CPython raises the direct exception coming
         # from the zlib module: let's do the same

From pypy.commits at gmail.com  Wed Feb 22 10:21:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 07:21:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58adac5d.c35c2e0a.28188.37e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90305:9eb20beecdd2
Date: 2017-02-22 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/9eb20beecdd2/

Log:	hg merge default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1601,16 +1601,6 @@
     def bytes_w(self, w_obj):
         return w_obj.bytes_w(self)
 
-    @not_rpython    # tests only; should be replaced with bytes0_w or text0_w
-    def str0_w(self, w_obj):
-        "Like str_w, but rejects strings with NUL bytes."
-        from rpython.rlib import rstring
-        result = self.str_w(w_obj)
-        if '\x00' in result:
-            raise oefmt(self.w_ValueError,
-                        "argument must be a string without NUL characters")
-        return rstring.assert_str0(result)
-
     def bytes0_w(self, w_obj):
         "Like bytes_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -206,11 +206,11 @@
         res = self.space.interp_w(Function, w(None), can_be_None=True)
         assert res is None
 
-    def test_str0_w(self):
+    def test_text0_w(self):
         space = self.space
         w = space.wrap
-        assert space.str0_w(w("123")) == "123"
-        exc = space.raises_w(space.w_ValueError, space.str0_w, w("123\x004"))
+        assert space.text0_w(w("123")) == "123"
+        exc = space.raises_w(space.w_ValueError, space.text0_w, w("123\x004"))
         assert space.unicode0_w(w(u"123")) == u"123"
         exc = space.raises_w(space.w_ValueError, space.unicode0_w, w(u"123\x004"))
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -95,12 +95,12 @@
         # The loop above sets space.w_str and space.w_bytes.
         # We rename 'space.w_str' to 'space.w_unicode' and
         # 'space.w_text'.
+        self.w_unicode = self.w_str
+        self.w_text = self.w_str
+        del self.w_str
         self.w_dict.flag_map_or_seq = 'M'
         self.w_list.flag_map_or_seq = 'S'
         self.w_tuple.flag_map_or_seq = 'S'
-        self.w_unicode = self.w_str
-        self.w_text = self.w_str
-        self.w_str = self.w_bytes      # BACKCOMPAT kill me
         self.builtin_types['str'] = self.w_unicode
         self.builtin_types['bytes'] = self.w_bytes
         self.builtin_types["NotImplemented"] = self.w_NotImplemented

From pypy.commits at gmail.com  Wed Feb 22 10:50:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 07:50:27 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Kill space.identifier_w(),
 which is almost not used any more.  We might
Message-ID: <58adb343.8c2b190a.3ecf2.2622@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90306:eaf70fb58a3e
Date: 2017-02-22 16:46 +0100
http://bitbucket.org/pypy/pypy/changeset/eaf70fb58a3e/

Log:	Kill space.identifier_w(), which is almost not used any more. We
	might need to think about surrogate-forbidding conversions, but
	later.

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -595,7 +595,7 @@
                         except IndexError:
                             name = '?'
                         else:
-                            name = space.identifier_w(w_name)
+                            name = space.text_w(w_name)
                     break
         self.kwd_name = name
 
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -433,7 +433,7 @@
         w_returns = get_field(space, w_node, 'returns', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.identifier_w(w_name)
+        _name = space.text_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         _args = arguments.from_object(space, w_args)
@@ -513,7 +513,7 @@
         w_returns = get_field(space, w_node, 'returns', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.identifier_w(w_name)
+        _name = space.text_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         _args = arguments.from_object(space, w_args)
@@ -606,7 +606,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.identifier_w(w_name)
+        _name = space.text_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         bases_w = space.unpackiterable(w_bases)
@@ -1511,7 +1511,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         names_w = space.unpackiterable(w_names)
-        _names = [space.identifier_w(w_item) for w_item in names_w]
+        _names = [space.text_w(w_item) for w_item in names_w]
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
@@ -1551,7 +1551,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         names_w = space.unpackiterable(w_names)
-        _names = [space.identifier_w(w_item) for w_item in names_w]
+        _names = [space.text_w(w_item) for w_item in names_w]
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Nonlocal(_names, _lineno, _col_offset)
@@ -2880,7 +2880,7 @@
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
-        _attr = space.identifier_w(w_attr)
+        _attr = space.text_w(w_attr)
         if _attr is None:
             raise_required_value(space, w_node, 'attr')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -3022,7 +3022,7 @@
         w_ctx = get_field(space, w_node, 'ctx', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _id = space.identifier_w(w_id)
+        _id = space.text_w(w_id)
         if _id is None:
             raise_required_value(space, w_node, 'id')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -3911,7 +3911,7 @@
         w_annotation = get_field(space, w_node, 'annotation', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _arg = space.identifier_w(w_arg)
+        _arg = space.text_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
         _annotation = expr.from_object(space, w_annotation)
@@ -3978,7 +3978,7 @@
     def from_object(space, w_node):
         w_name = get_field(space, w_node, 'name', False)
         w_asname = get_field(space, w_node, 'asname', True)
-        _name = space.identifier_w(w_name)
+        _name = space.text_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         _asname = space.text_or_none_w(w_asname)
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -162,7 +162,7 @@
         elif field.type in ("identifier",):
             if field.opt:
                 return "space.text_or_none_w(%s)" % (value,)
-            return "space.identifier_w(%s)" % (value,)
+            return "space.text_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
         elif field.type in ("bool",):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -236,9 +236,6 @@
     def unicode_w(self, space):
         self._typed_unwrap_error(space, "string")
 
-    def identifier_w(self, space):
-        self._typed_unwrap_error(space, "string")
-
     def text_w(self, space):
         self._typed_unwrap_error(space, "string")
 
@@ -1522,8 +1519,8 @@
             # most API in CPython 3.x no longer do.
             if self.isinstance_w(w_obj, self.w_bytes):
                 return StringBuffer(w_obj.bytes_w(self))
-            if self.isinstance_w(w_obj, self.w_unicode):
-                return StringBuffer(w_obj.identifier_w(self))  # no surrogates
+            if self.isinstance_w(w_obj, self.w_unicode):  # NB. CPython forbids
+                return StringBuffer(w_obj.text_w(self))   # surrogates here
             try:
                 return w_obj.buffer_w(self, self.BUF_SIMPLE)
             except BufferInterfaceNotFound:
@@ -1534,8 +1531,8 @@
             # most API in CPython 3.x no longer do.
             if self.isinstance_w(w_obj, self.w_bytes):
                 return w_obj.bytes_w(self)
-            if self.isinstance_w(w_obj, self.w_unicode):
-                return w_obj.identifier_w(self)    # no surrogates (forbidden)
+            if self.isinstance_w(w_obj, self.w_unicode):  # NB. CPython forbids
+                return w_obj.text_w(self)                 # surrogates here
             try:
                 return w_obj.buffer_w(self, self.BUF_SIMPLE).as_str()
             except BufferInterfaceNotFound:
@@ -1590,8 +1587,7 @@
         encoded string). Else, call bytes_w().
 
         We should kill str_w completely and manually substitute it with
-        text_w/identifier_w/bytes_w at all call sites.  It remains for
-        now for tests only.
+        text_w/bytes_w at all call sites.  It remains for now for tests only.
         """
         if self.isinstance_w(w_obj, self.w_unicode):
             return w_obj.text_w(self)
@@ -1680,18 +1676,6 @@
     realtext_w = text_w         # Python 2 compatibility
     realunicode_w = unicode_w
 
-    def identifier_w(self, w_obj):
-        """
-        Unwrap an object which is used as an identifier (i.e. names of
-        variables, methdods, functions, classes etc.). In py3k, identifiers
-        are unicode strings and are unwrapped as UTF-8 encoded byte strings.
-        This differs from space.text_w() because it raises an app-level
-        UnicodeEncodeError if the unicode string contains surrogates.
-        This corresponds exactly to 'str.encode(obj, "utf-8")' at app-level.
-        (XXX check what occurs on narrow builds or kill narrow builds!)
-        """
-        return w_obj.identifier_w(self)
-
     def fsencode(space, w_obj):
         from pypy.interpreter.unicodehelper import fsencode
         return fsencode(space, w_obj)
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -97,17 +97,12 @@
     newtext = wrap
     newunicode = wrap
 
-    def str_w(self, s):
-        return str(s)
     def text_w(self, s):
-        return self.str_w(s)
+        return self.unicode_w(s).encode('utf-8')
 
     def unicode_w(self, s):
         return unicode(s)
 
-    def identifier_w(self, s):
-        return self.unicode_w(s).encode('utf-8')
-
     def len(self, x):
         return len(x)
 
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -214,11 +214,11 @@
         assert space.unicode0_w(w(u"123")) == u"123"
         exc = space.raises_w(space.w_ValueError, space.unicode0_w, w(u"123\x004"))
 
-    def test_identifier_w(self):
+    def test_text_w(self):
         space = self.space
         x = u'&#224;&#232;&#236;'
         w_name = space.wrap(x)
-        assert space.identifier_w(w_name) == x.encode('utf-8')
+        assert space.text_w(w_name) == x.encode('utf-8')
 
     def test_getindex_w(self):
         w_instance1 = self.space.appexec([], """():
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -45,7 +45,7 @@
 
     def text_w(self, space):
         return NonConstant("foobar")
-    identifier_w = bytes_w = text_w
+    bytes_w = text_w
 
     def unicode_w(self, space):
         return NonConstant(u"foobar")
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -30,13 +30,11 @@
                 space.w_unicode, "__new__", space.w_unicode, w_uni)
         assert w_new is w_uni
 
-    def test_identifier_or_text_w(self):
+    def test_text_w(self):
         space = self.space
         w_uni = space.wrap(u'abcd')
-        assert space.identifier_w(w_uni) == 'abcd'
         assert space.text_w(w_uni) == 'abcd'
         w_uni = space.wrap(unichr(0xd921) + unichr(0xdddd))
-        space.raises_w(space.w_UnicodeEncodeError, space.identifier_w, w_uni)
         assert space.text_w(w_uni) == '\xed\xa4\xa1\xed\xb7\x9d'
         #                             ^^^ and not the 4-bytes combined character
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -8,8 +8,7 @@
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii,
-    unicode_encode_utf8sp, unicode_encode_utf8_forbid_surrogates,
-    SurrogateError)
+    unicode_encode_utf8sp)
 from rpython.rlib import jit
 
 from pypy.interpreter import unicodehelper
@@ -81,32 +80,13 @@
     def unicode_w(self, space):
         return self._value
 
-    def _identifier_or_text_w(self, space, ignore_sg):
-        try:
-            identifier = jit.conditional_call_elidable(
-                                self._utf8, g_encode_utf8, self._value)
-            if not jit.isconstant(self):
-                self._utf8 = identifier
-        except SurrogateError:
-            # If 'ignore_sg' is False, this logic is here only
-            # to get an official app-level UnicodeEncodeError.
-            # If 'ignore_sg' is True, we encode instead using
-            # unicode_encode_utf8sp().
-            u = self._value
-            if ignore_sg:
-                identifier = unicode_encode_utf8sp(u, len(u))
-            else:
-                eh = unicodehelper.encode_error_handler(space)
-                identifier = unicode_encode_utf_8(u, len(u), None,
-                                                  errorhandler=eh)
+    def text_w(self, space):
+        identifier = jit.conditional_call_elidable(
+                            self._utf8, g_encode_utf8, self._value)
+        if not jit.isconstant(self):
+            self._utf8 = identifier
         return identifier
 
-    def text_w(self, space):
-        return self._identifier_or_text_w(space, ignore_sg=True)
-
-    def identifier_w(self, space):
-        return self._identifier_or_text_w(space, ignore_sg=False)
-
     def listview_unicode(self):
         return _create_list_from_unicode(self._value)
 
@@ -1277,7 +1257,7 @@
 @jit.elidable
 def g_encode_utf8(value):
     """This is a global function because of jit.conditional_call_value"""
-    return unicode_encode_utf8_forbid_surrogates(value, len(value))
+    return unicode_encode_utf8sp(value, len(value))
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')

From pypy.commits at gmail.com  Wed Feb 22 10:58:31 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Feb 2017 07:58:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: call identifier_w on first param of call
 type("A\udcdcb", (), {}), instead of text_w. do not ignore surrogate error
Message-ID: <58adb527.99012e0a.951b8.e7a9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90307:d7116d6e5a29
Date: 2017-02-22 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/d7116d6e5a29/

Log:	call identifier_w on first param of call type("A\udcdcb", (), {}),
	instead of text_w. do not ignore surrogate error

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1471,3 +1471,6 @@
         #
         assert type.__dict__['__name__'].__name__ == '__name__'
         assert type.__dict__['__doc__'].__name__ == '__doc__'
+
+    def test_type_construct_unicode_surrogate_issue(self):
+        raises(ValueError, type, 'A\udcdcb', (), {})
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -724,7 +724,7 @@
             return space.call_function(newfunc, w_winner, w_name, w_bases, w_dict)
         w_typetype = w_winner
 
-    name = space.text_w(w_name)
+    name = space.identifier_w(w_name)
     assert isinstance(name, str)
     if '\x00' in name:
         raise oefmt(space.w_ValueError, "type name must not contain null characters")

From pypy.commits at gmail.com  Wed Feb 22 11:05:32 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Feb 2017 08:05:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: revert back to text_w and add comment
Message-ID: <58adb6cc.18532e0a.7645f.d4e1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90308:342121c5c4e5
Date: 2017-02-22 17:04 +0100
http://bitbucket.org/pypy/pypy/changeset/342121c5c4e5/

Log:	revert back to text_w and add comment

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -724,7 +724,7 @@
             return space.call_function(newfunc, w_winner, w_name, w_bases, w_dict)
         w_typetype = w_winner
 
-    name = space.identifier_w(w_name)
+    name = space.text_w(w_name) # NB. CPython forbids surrogates here
     assert isinstance(name, str)
     if '\x00' in name:
         raise oefmt(space.w_ValueError, "type name must not contain null characters")

From pypy.commits at gmail.com  Wed Feb 22 12:02:47 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 22 Feb 2017 09:02:47 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: start passing some unicode tests.
 no UCS4 support yet completely
Message-ID: <58adc437.1c152e0a.d64cd.2c45@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90309:f05bed30187f
Date: 2017-02-22 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/f05bed30187f/

Log:	start passing some unicode tests. no UCS4 support yet completely

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -58,7 +58,8 @@
     # you still get two surrogate unicode characters in the result.
     # These are the Python2 rules; Python3 differs.
     consumed, length = rutf8.str_check_utf8(
-        string, "strict", final=True, errorhandler=decode_error_handler(space),
+        string, len(string), "strict", final=True,
+        errorhandler=decode_error_handler(space),
         allow_surrogates=True)
     return length
 
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -16,7 +16,7 @@
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.unicodeobject import (
-    decode_object, utf8_from_encoded_object,
+    decode_object, unicode_from_encoded_object,
     getdefaultencoding)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
@@ -717,7 +717,7 @@
             self_as_unicode = unicode_from_encoded_object(space, self, None,
                                                           None)
             return space.newbool(
-                self_as_unicode._value.find(w_sub._value) >= 0)
+                self_as_unicode._utf8.find(w_sub._utf8) >= 0)
         return self._StringMethods_descr_contains(space, w_sub)
 
     _StringMethods_descr_replace = descr_replace
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -94,7 +94,7 @@
         else:
             return space.fromcache(BytesListStrategy)
 
-    elif type(w_firstobj) is W_UnicodeObject:
+    elif False and type(w_firstobj) is W_UnicodeObject: # disable unicode list strat
         # check for all-unicodes
         for i in range(1, len(list_w)):
             if type(list_w[i]) is not W_UnicodeObject:
@@ -195,6 +195,7 @@
 
     @staticmethod
     def newlist_unicode(space, list_u):
+        xxxx
         strategy = space.fromcache(UnicodeListStrategy)
         storage = strategy.erase(list_u)
         return W_ListObject.from_storage_and_strategy(space, storage, strategy)
@@ -958,8 +959,8 @@
             strategy = self.space.fromcache(IntegerListStrategy)
         elif type(w_item) is W_BytesObject:
             strategy = self.space.fromcache(BytesListStrategy)
-        elif type(w_item) is W_UnicodeObject:
-            strategy = self.space.fromcache(UnicodeListStrategy)
+        #elif type(w_item) is W_UnicodeObject:
+        #    strategy = self.space.fromcache(UnicodeListStrategy)
         elif type(w_item) is W_FloatObject:
             strategy = self.space.fromcache(FloatListStrategy)
         else:
@@ -2005,7 +2006,7 @@
         return self.space.newunicode(stringval)
 
     def unwrap(self, w_string):
-        return self.space.unicode_w(w_string)
+        return self.space.utf8_w(w_string)
 
     erase, unerase = rerased.new_erasing_pair("unicode")
     erase = staticmethod(erase)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -299,6 +299,7 @@
     newlist_text = newlist_bytes
 
     def newlist_unicode(self, list_u):
+        return self.newlist(list_u)
         return W_ListObject.newlist_unicode(self, list_u)
 
     def newlist_int(self, list_i):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -2,13 +2,13 @@
 
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin,
-    enforceargs)
+    enforceargs, newlist_hint)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
-from rpython.rlib import rutf8
+from rpython.rlib import rutf8, jit
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
@@ -110,8 +110,8 @@
                          "found", len(self._value))
         return space.newint(ord(self._value[0]))
 
-    def _new(self, value):
-        return W_UnicodeObject(value)
+    def _new(self, value, length):
+        return W_UnicodeObject(value, length)
 
     def _new_from_list(self, value):
         return W_UnicodeObject(u''.join(value))
@@ -120,7 +120,7 @@
         return W_UnicodeObject.EMPTY
 
     def _len(self):
-        return len(self._value)
+        return self._length
 
     _val = utf8_w
 
@@ -135,18 +135,25 @@
         if isinstance(w_other, W_UnicodeObject):
             return w_other._utf8
         if space.isinstance_w(w_other, space.w_bytes):
-            return utf8_from_string(space, w_other)._utf8
+            return unicode_from_string(space, w_other)._utf8
         if strict:
             raise oefmt(space.w_TypeError,
                 "%s arg must be None, unicode or str", strict)
-        return utf8_from_encoded_object(
-            space, w_other, None, "strict")._value
+        return unicode_from_encoded_object(
+            space, w_other, None, "strict")._utf8
+
+    def _convert_to_unicode(self, space, w_other):
+        if isinstance(w_other, W_UnicodeObject):
+            return w_other
+        if space.isinstance_w(w_other, space.w_bytes):
+            return unicode_from_string(space, w_other)
+        return unicode_from_encoded_object(space, w_other, None, "strict")
 
     def _chr(self, char):
         assert len(char) == 1
         return unicode(char)[0]
 
-    _builder = UnicodeBuilder
+    _builder = StringBuilder
 
     def _isupper(self, ch):
         return unicodedb.isupper(ord(ch))
@@ -423,6 +430,46 @@
     def _starts_ends_overflow(self, prefix):
         return len(prefix) == 0
 
+    def descr_add(self, space, w_other):
+        try:
+            w_other = self._convert_to_unicode(space, w_other)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
+        return W_UnicodeObject(self._utf8 + w_other._utf8,
+                               self._length + w_other._length)
+
+    @jit.look_inside_iff(lambda self, space, list_w, size:
+                         jit.loop_unrolling_heuristic(list_w, size))
+    def _str_join_many_items(self, space, list_w, size):
+        value = self._utf8
+        lgt = self._length * (size - 1)
+
+        prealloc_size = len(value) * (size - 1)
+        unwrapped = newlist_hint(size)
+        for i in range(size):
+            w_s = list_w[i]
+            check_item = self._join_check_item(space, w_s)
+            if check_item == 1:
+                raise oefmt(space.w_TypeError,
+                            "sequence item %d: expected string, %T found",
+                            i, w_s)
+            elif check_item == 2:
+                return self._join_autoconvert(space, list_w)
+            # XXX Maybe the extra copy here is okay? It was basically going to
+            #     happen anyway, what with being placed into the builder
+            w_u = self._convert_to_unicode(space, w_s)
+            unwrapped.append(w_u._utf8)
+            lgt += w_u._length
+            prealloc_size += len(unwrapped[i])
+
+        sb = self._builder(prealloc_size)
+        for i in range(size):
+            if value and i != 0:
+                sb.append(value)
+            sb.append(unwrapped[i])
+        return self._new(sb.build(), lgt)
 
 def wrapunicode(space, uni):
     return W_UnicodeObject(uni)
@@ -515,7 +562,7 @@
                 unicodehelper.decode_error_handler(space)(None,
                     'ascii', "ordinal not in range(128)", s, e.pos, e.pos+1)
                 assert False
-            return space.newunicode(s)
+            return space.newunicode(s, len(s))
         if encoding == 'utf-8':
             yyy
             s = space.charbuf_w(w_obj)
@@ -534,7 +581,7 @@
     return w_retval
 
 
-def utf8_from_encoded_object(space, w_obj, encoding, errors):
+def unicode_from_encoded_object(space, w_obj, encoding, errors):
     # explicitly block bytearray on 2.7
     from .bytearrayobject import W_BytearrayObject
     if isinstance(w_obj, W_BytearrayObject):
@@ -571,7 +618,7 @@
     return unicode_from_encoded_object(space, w_res, None, "strict")
 
 
-def utf8_from_string(space, w_bytes):
+def unicode_from_string(space, w_bytes):
     # this is a performance and bootstrapping hack
     encoding = getdefaultencoding(space)
     if encoding != 'ascii':
@@ -582,7 +629,7 @@
         rutf8.check_ascii(s)
     except rutf8.AsciiCheckError:
         # raising UnicodeDecodeError is messy, "please crash for me"
-        return utf8_from_encoded_object(space, w_bytes, "ascii", "strict")
+        return unicode_from_encoded_object(space, w_bytes, "ascii", "strict")
     return W_UnicodeObject(s, len(s))
 
 
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -243,7 +243,7 @@
         errorhandler = default_unicode_error_decode
 
     if size == 0:
-        return '', 0
+        return '', 0, 0
 
     lgt = 0
     builder = StringBuilder(size)

From pypy.commits at gmail.com  Wed Feb 22 12:09:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 22 Feb 2017 09:09:12 -0800 (PST)
Subject: [pypy-commit] pypy default: oops, translation fix
Message-ID: <58adc5b8.8bd4190a.4625e.2ba5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90310:9de068384802
Date: 2017-02-22 17:08 +0000
http://bitbucket.org/pypy/pypy/changeset/9de068384802/

Log:	oops, translation fix

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1318,7 +1318,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.realtext_w(w_module) if not space.is_none(w_module) else space.w_None
+        _module = space.realtext_w(w_module) if not space.is_none(w_module) else None
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -3415,8 +3415,8 @@
         w_defaults = get_field(space, w_node, 'defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [expr.from_object(space, w_item) for w_item in args_w]
-        _vararg = space.realtext_w(w_vararg) if not space.is_none(w_vararg) else space.w_None
-        _kwarg = space.realtext_w(w_kwarg) if not space.is_none(w_kwarg) else space.w_None
+        _vararg = space.realtext_w(w_vararg) if not space.is_none(w_vararg) else None
+        _kwarg = space.realtext_w(w_kwarg) if not space.is_none(w_kwarg) else None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwarg, _defaults)
@@ -3485,7 +3485,7 @@
         _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.realtext_w(w_asname) if not space.is_none(w_asname) else space.w_None
+        _asname = space.realtext_w(w_asname) if not space.is_none(w_asname) else None
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -151,7 +151,7 @@
         elif field.type in ("identifier",):
             if field.opt:
                 return ("space.realtext_w(%s) if not space.is_none(%s) "
-                        "else space.w_None" % (value, value))
+                        "else None" % (value, value))
             return "space.realtext_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)

From pypy.commits at gmail.com  Wed Feb 22 12:31:26 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 22 Feb 2017 09:31:26 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: rename newunicode to utf8
Message-ID: <58adcaee.65012e0a.ce6bc.e0c4@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90311:16e1ade09c1c
Date: 2017-02-22 18:31 +0100
http://bitbucket.org/pypy/pypy/changeset/16e1ade09c1c/

Log:	rename newunicode to utf8

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -63,7 +63,7 @@
             v, length = unicodehelper.decode_raw_unicode_escape(space, substr)
         else:
             v, length = unicodehelper.decode_unicode_escape(space, substr)
-        return space.newunicode(v, length)
+        return space.newutf8(v, length)
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "utf8" and
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -30,7 +30,7 @@
         s, lgt = unichr_as_utf8(code)
     except ValueError:
         raise oefmt(space.w_ValueError, "unichr() arg out of range")
-    return space.newunicode(s, lgt)
+    return space.newutf8(s, lgt)
 
 def len(space, w_obj):
     "len(object) -> integer\n\nReturn the number of items of a sequence or mapping."
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -405,7 +405,7 @@
 def unmarshal_unicode(space, u, tc):
     arg = u.get_str()
     length = unicodehelper.check_utf8(space, arg)
-    return space.newunicode(arg, length)
+    return space.newutf8(arg, length)
 
 @marshaller(W_SetObject)
 def marshal_set(space, w_set, m):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -347,7 +347,7 @@
             return self.w_None
         return self.newtext(s)
 
-    def newunicode(self, utf8s, length):
+    def newutf8(self, utf8s, length):
         assert utf8s is not None
         assert isinstance(utf8s, str)
         return W_UnicodeObject(utf8s, length)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -562,7 +562,7 @@
                 unicodehelper.decode_error_handler(space)(None,
                     'ascii', "ordinal not in range(128)", s, e.pos, e.pos+1)
                 assert False
-            return space.newunicode(s, len(s))
+            return space.newutf8(s, len(s))
         if encoding == 'utf-8':
             yyy
             s = space.charbuf_w(w_obj)

From pypy.commits at gmail.com  Wed Feb 22 13:51:35 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 22 Feb 2017 10:51:35 -0800 (PST)
Subject: [pypy-commit] pypy default: Check for errors raised in sq_length
Message-ID: <58adddb7.4fd0190a.58a88.3212@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90312:ba83efb4837f
Date: 2017-02-22 18:51 +0000
http://bitbucket.org/pypy/pypy/changeset/ba83efb4837f/

Log:	Check for errors raised in sq_length

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -237,7 +237,10 @@
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
     check_num_args(space, w_args, 0)
-    return space.newint(generic_cpy_call(space, func_len, w_self))
+    res = generic_cpy_call(space, func_len, w_self)
+    if widen(res) == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.newint(res)
 
 def wrap_sq_item(space, w_self, w_args, func):
     func_target = rffi.cast(ssizeargfunc, func)

From pypy.commits at gmail.com  Wed Feb 22 14:53:54 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 22 Feb 2017 11:53:54 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-3: document branch
Message-ID: <58adec52.8e48190a.65b34.34ee@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-3
Changeset: r90313:8fa25d257df0
Date: 2017-02-22 20:53 +0100
http://bitbucket.org/pypy/pypy/changeset/8fa25d257df0/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -142,3 +142,9 @@
 it is the standard ``OrderedDict.move_to_end()`` method, but the
 behavior is also available on Python 2.x or for the ``dict`` type by
 calling ``__pypy__.move_to_end(dict, key, last=True)``.
+
+
+.. branch optinfo-into-bridges-3
+
+Improve the optimization of branchy Python code by retaining more information
+across failing guards.

From pypy.commits at gmail.com  Wed Feb 22 16:13:56 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 22 Feb 2017 13:13:56 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-3: merge default
Message-ID: <58adff14.0b202e0a.93a0a.f5a7@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-3
Changeset: r90314:8404d10c4bc1
Date: 2017-02-22 20:55 +0100
http://bitbucket.org/pypy/pypy/changeset/8404d10c4bc1/

Log:	merge default

diff too long, truncating to 2000 out of 28813 lines

diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,9 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break        
+                break
+        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+                                                 'include')
 
     if args:
         vals = []
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -604,7 +604,8 @@
         """
         # hack for performance: if restype is a "simple" primitive type, don't
         # allocate the buffer because it's going to be thrown away immediately
-        if self._is_primitive(restype) and not restype._is_pointer_like():
+        if (self._is_primitive(restype) and restype._type_ != '?'
+            and not restype._is_pointer_like()):
             return result
         #
         shape = restype._ffishape_
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -112,11 +112,20 @@
     def _make_cmp(name):
         cmpfunc = getattr(operator, name)
         def cmp(self, other):
-            if isinstance(other, CTypesData):
+            v_is_ptr = not isinstance(self, CTypesGenericPrimitive)
+            w_is_ptr = (isinstance(other, CTypesData) and
+                           not isinstance(other, CTypesGenericPrimitive))
+            if v_is_ptr and w_is_ptr:
                 return cmpfunc(self._convert_to_address(None),
                                other._convert_to_address(None))
+            elif v_is_ptr or w_is_ptr:
+                return NotImplemented
             else:
-                return NotImplemented
+                if isinstance(self, CTypesGenericPrimitive):
+                    self = self._value
+                if isinstance(other, CTypesGenericPrimitive):
+                    other = other._value
+                return cmpfunc(self, other)
         cmp.func_name = name
         return cmp
 
@@ -128,7 +137,7 @@
     __ge__ = _make_cmp('__ge__')
 
     def __hash__(self):
-        return hash(type(self)) ^ hash(self._convert_to_address(None))
+        return hash(self._convert_to_address(None))
 
     def _to_string(self, maxlen):
         raise TypeError("string(): %r" % (self,))
@@ -137,14 +146,8 @@
 class CTypesGenericPrimitive(CTypesData):
     __slots__ = []
 
-    def __eq__(self, other):
-        return self is other
-
-    def __ne__(self, other):
-        return self is not other
-
     def __hash__(self):
-        return object.__hash__(self)
+        return hash(self._value)
 
     def _get_own_repr(self):
         return repr(self._from_ctypes(self._value))
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -188,6 +188,7 @@
 
 .. py:function:: wrap(x)
 
+   **Deprecated! Eventually this method should disappear.**
    Returns a wrapped object that is a reference to the interpreter-level object
    :py:obj:`x`. This can be used either on simple immutable objects (integers,
    strings, etc) to create a new wrapped object, or on instances of :py:class:`W_Root`
@@ -196,6 +197,35 @@
    be directly exposed to application-level code in this way - functions, frames,
    code objects, etc.
 
+.. py:function:: newint(i)
+
+   Creates a wrapped object holding an integral value. `newint` creates an object
+   of type `W_IntObject`.
+
+.. py:function:: newlong(l)
+
+   Creates a wrapped object holding an integral value. The main difference to newint
+   is the type of the argument (which is rpython.rlib.rbigint.rbigint). On PyPy3 this
+   method will return an :py:class:`int` (PyPy2 it returns a :py:class:`long`).
+
+.. py:function:: newbytes(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`bytes` (both on PyPy2 and PyPy3).
+
+.. py:function:: newtext(t)
+
+   The given argument is a rpython bytestring. Creates a wrapped object
+   of type :py:class:`str`.  On PyPy3 this will return a wrapped unicode
+   object. The object will hold a utf-8-nosg decoded value of `t`.
+   The "utf-8-nosg" codec used here is slightly different from the
+   "utf-8" implemented in Python 2 or Python 3: it is defined as utf-8
+   without any special handling of surrogate characters.  They are
+   encoded using the same three-bytes sequence that encodes any char in
+   the range from ``'\u0800'`` to ``'\uffff'``.
+
+   PyPy2 will return a bytestring object. No encoding/decoding steps will be applied.
+
 .. py:function:: newbool(b)
 
    Creates a wrapped :py:class:`bool` object from an :ref:`interpreter-level <interpreter-level>`
@@ -217,15 +247,18 @@
 
    Creates a new slice object.
 
-.. py:function:: newstring(asciilist)
+.. py:function:: newunicode(ustr)
 
-   Creates a string from a list of wrapped integers. Note that this may not be
-   a very useful method; usually you can just write ``space.wrap("mystring")``.
+   Creates a Unicode string from an rpython unicode string.
+   This method may disappear soon and be replaced by :py:function:`newutf8()`.
 
-.. py:function:: newunicode(codelist)
+.. py:function:: newutf8(bytestr)
 
-   Creates a Unicode string from a list of integers (code points).
+   Creates a Unicode string from an rpython byte string, decoded as
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
 
+Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
+`pypy/objspace/std/objspace.py`.
 
 Conversions from Application Level to Interpreter Level
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -258,11 +291,28 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: text_w(w_x)
+
+   Takes an application level :py:class:`str` and converts it to a rpython byte string.
+   PyPy3 this method will return an utf-8-nosg encoded result.
+
+.. py:function:: bytes_w(w_x)
+
+   Takes an application level :py:class:`bytes` (PyPy2 this equals `str`) and returns a rpython
+   byte string.
+
 .. py:function:: str_w(w_x)
 
+   **Deprecated. use text_w or bytes_w instead**
    If :py:obj:`w_x` is an application-level string, return an interpreter-level string.
    Otherwise raise :py:exc:`TypeError`.
 
+.. py:function:: unicode_w(w_x)
+
+   Takes an application level :py:class:`unicode` and return an
+   interpreter-level unicode string.  This method may disappear soon and
+   be replaced by :py:function:`text_w()`.
+
 .. py:function:: float_w(w_x)
 
    If :py:obj:`w_x` is an application-level float, integer or long, return an
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -148,3 +148,16 @@
 
 Improve the optimization of branchy Python code by retaining more information
 across failing guards.
+
+
+.. branch: space-newtext
+
+Internal refactoring of ``space.wrap()``, which is now replaced with
+explicitly-typed methods.  Notably, there are now ``space.newbytes()``
+and ``space.newtext()``: these two methods are identical on PyPy 2.7 but
+not on PyPy 3.x.  The latter is used to get an app-level unicode string
+by decoding the RPython string, assumed to be utf-8.
+
+.. branch: fix_bool_restype
+
+Fix for ``ctypes.c_bool``-returning ctypes functions
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -31,9 +31,9 @@
 
 def create_entry_point(space, w_dict):
     if w_dict is not None: # for tests
-        w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
-        w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
-        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
+        w_entry_point = space.getitem(w_dict, space.newtext('entry_point'))
+        w_run_toplevel = space.getitem(w_dict, space.newtext('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.newtext('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
         hashfunc = space.config.objspace.hash
     else:
@@ -65,8 +65,8 @@
         try:
             try:
                 space.startup()
-                w_executable = space.wrap(argv[0])
-                w_argv = space.newlist([space.wrap(s) for s in argv[1:]])
+                w_executable = space.newtext(argv[0])
+                w_argv = space.newlist([space.newtext(s) for s in argv[1:]])
                 w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
                 exitcode = space.int_w(w_exitcode)
                 # try to pull it all in
@@ -76,7 +76,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
             try:
@@ -84,7 +84,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
 
@@ -123,7 +123,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap(home), w_initstdio],
+            space.appexec([w_path, space.newtext(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path
@@ -141,7 +141,7 @@
             if verbose:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
             if must_leave:
@@ -180,11 +180,11 @@
     def _pypy_execute_source(source, c_argument):
         try:
             w_globals = space.newdict(module=True)
-            space.setitem(w_globals, space.wrap('__builtins__'),
+            space.setitem(w_globals, space.newtext('__builtins__'),
                           space.builtin_modules['__builtin__'])
-            space.setitem(w_globals, space.wrap('c_argument'),
-                          space.wrap(c_argument))
-            space.appexec([space.wrap(source), w_globals], """(src, glob):
+            space.setitem(w_globals, space.newtext('c_argument'),
+                          space.newint(c_argument))
+            space.appexec([space.newtext(source), w_globals], """(src, glob):
                 import sys
                 stmt = compile(src, 'c callback', 'exec')
                 if not hasattr(sys, '_pypy_execute_source'):
@@ -195,7 +195,7 @@
         except OperationError as e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space))
-            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
 
@@ -323,7 +323,7 @@
         # obscure hack to stuff the translation options into the translated PyPy
         import pypy.module.sys
         options = make_dict(config)
-        wrapstr = 'space.wrap(%r)' % (options)
+        wrapstr = 'space.wrap(%r)' % (options) # import time
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -363,7 +363,7 @@
     i = 0
     for w_key in keys_w:
         try:
-            key = space.str_w(w_key)
+            key = space.text_w(w_key)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError, "keywords must be strings")
@@ -547,11 +547,11 @@
                         except IndexError:
                             name = '?'
                         else:
-                            w_enc = space.wrap(space.sys.defaultencoding)
-                            w_err = space.wrap("replace")
+                            w_enc = space.newtext(space.sys.defaultencoding)
+                            w_err = space.newtext("replace")
                             w_name = space.call_method(w_name, "encode", w_enc,
                                                        w_err)
-                            name = space.str_w(w_name)
+                            name = space.text_w(w_name)
                     break
         self.kwd_name = name
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -266,8 +266,8 @@
             else:
                 w_key = space.newtuple([obj, space.w_float])
         elif space.is_w(w_type, space.w_complex):
-            w_real = space.getattr(obj, space.wrap("real"))
-            w_imag = space.getattr(obj, space.wrap("imag"))
+            w_real = space.getattr(obj, space.newtext("real"))
+            w_imag = space.getattr(obj, space.newtext("imag"))
             real = space.float_w(w_real)
             imag = space.float_w(w_imag)
             real_negzero = (real == 0.0 and
@@ -366,7 +366,7 @@
         space = self.space
         consts_w = [space.w_None] * space.len_w(w_consts)
         w_iter = space.iter(w_consts)
-        first = space.wrap(0)
+        first = space.newint(0)
         while True:
             try:
                 w_key = space.next(w_iter)
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -13,7 +13,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -48,10 +48,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
-
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+        assert fields == []
+
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -67,14 +67,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -93,12 +93,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -114,7 +114,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -143,16 +143,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)
+            space.newtext(name), space.newtuple([w_base]), w_dict)
         setattr(self, 'w_%s' % name, w_type)
 
 def get(space):
@@ -196,7 +196,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -230,7 +230,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -258,7 +258,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expression)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -293,7 +293,7 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         return w_node
 
     @staticmethod
@@ -390,26 +390,26 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_FunctionDef)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -420,7 +420,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         _args = arguments.from_object(space, w_args)
@@ -463,30 +463,30 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ClassDef)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.bases is None:
             bases_w = []
         else:
             bases_w = [node.to_object(space) for node in self.bases] # expr
         w_bases = space.newlist(bases_w)
-        space.setattr(w_node, space.wrap('bases'), w_bases)
+        space.setattr(w_node, space.newtext('bases'), w_bases)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.decorator_list is None:
             decorator_list_w = []
         else:
             decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr
         w_decorator_list = space.newlist(decorator_list_w)
-        space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -497,7 +497,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         bases_w = space.unpackiterable(w_bases)
@@ -530,11 +530,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Return)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -572,11 +572,11 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -617,13 +617,13 @@
         else:
             targets_w = [node.to_object(space) for node in self.targets] # expr
         w_targets = space.newlist(targets_w)
-        space.setattr(w_node, space.wrap('targets'), w_targets)
+        space.setattr(w_node, space.newtext('targets'), w_targets)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -663,15 +663,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_AugAssign)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -719,19 +719,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Print)
         w_dest = self.dest.to_object(space) if self.dest is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('dest'), w_dest)
+        space.setattr(w_node, space.newtext('dest'), w_dest)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_nl = space.wrap(self.nl)  # bool
-        space.setattr(w_node, space.wrap('nl'), w_nl)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_nl = space.newbool(self.nl)  # bool
+        space.setattr(w_node, space.newtext('nl'), w_nl)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -780,25 +780,25 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_For)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -850,23 +850,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_While)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -914,23 +914,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_If)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -977,19 +977,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_With)
         w_context_expr = self.context_expr.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('context_expr'), w_context_expr)
+        space.setattr(w_node, space.newtext('context_expr'), w_context_expr)
         w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars)
+        space.setattr(w_node, space.newtext('optional_vars'), w_optional_vars)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1035,15 +1035,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Raise)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
+        space.setattr(w_node, space.newtext('type'), w_type)
         w_inst = self.inst.to_object(space) if self.inst is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('inst'), w_inst)
+        space.setattr(w_node, space.newtext('inst'), w_inst)
         w_tback = self.tback.to_object(space) if self.tback is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('tback'), w_tback)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('tback'), w_tback)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1093,23 +1093,23 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.handlers is None:
             handlers_w = []
         else:
             handlers_w = [node.to_object(space) for node in self.handlers] # excepthandler
         w_handlers = space.newlist(handlers_w)
-        space.setattr(w_node, space.wrap('handlers'), w_handlers)
+        space.setattr(w_node, space.newtext('handlers'), w_handlers)
         if self.orelse is None:
             orelse_w = []
         else:
             orelse_w = [node.to_object(space) for node in self.orelse] # stmt
         w_orelse = space.newlist(orelse_w)
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1158,17 +1158,17 @@
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         if self.finalbody is None:
             finalbody_w = []
         else:
             finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt
         w_finalbody = space.newlist(finalbody_w)
-        space.setattr(w_node, space.wrap('finalbody'), w_finalbody)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('finalbody'), w_finalbody)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1207,13 +1207,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Assert)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_msg = self.msg.to_object(space) if self.msg is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('msg'), w_msg)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('msg'), w_msg)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1255,11 +1255,11 @@
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1295,20 +1295,20 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ImportFrom)
-        w_module = space.wrap(self.module)  # identifier
-        space.setattr(w_node, space.wrap('module'), w_module)
+        w_module = space.newtext_or_none(self.module)  # identifier
+        space.setattr(w_node, space.newtext('module'), w_module)
         if self.names is None:
             names_w = []
         else:
             names_w = [node.to_object(space) for node in self.names] # alias
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_level = space.wrap(self.level)  # int
-        space.setattr(w_node, space.wrap('level'), w_level)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_level = space.newint(self.level)  # int
+        space.setattr(w_node, space.newtext('level'), w_level)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1318,7 +1318,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.str_or_None_w(w_module)
+        _module = space.realtext_w(w_module) if not space.is_none(w_module) else None
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -1351,15 +1351,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Exec)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_globals = self.globals.to_object(space) if self.globals is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('globals'), w_globals)
+        space.setattr(w_node, space.newtext('globals'), w_globals)
         w_locals = self.locals.to_object(space) if self.locals is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('locals'), w_locals)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('locals'), w_locals)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1398,13 +1398,13 @@
         if self.names is None:
             names_w = []
         else:
-            names_w = [space.wrap(node) for node in self.names] # identifier
+            names_w = [space.newtext(node) for node in self.names] # identifier
         w_names = space.newlist(names_w)
-        space.setattr(w_node, space.wrap('names'), w_names)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('names'), w_names)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1413,7 +1413,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         names_w = space.unpackiterable(w_names)
-        _names = [space.realstr_w(w_item) for w_item in names_w]
+        _names = [space.realtext_w(w_item) for w_item in names_w]
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
@@ -1437,11 +1437,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Expr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1472,10 +1472,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Pass)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1502,10 +1502,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Break)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1532,10 +1532,10 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Continue)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1628,17 +1628,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BoolOp)
         w_op = boolop_to_class[self.op - 1]().to_object(space)  # boolop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1678,15 +1678,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_BinOp)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         w_op = operator_to_class[self.op - 1]().to_object(space)  # operator
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_right = self.right.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('right'), w_right)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('right'), w_right)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1729,13 +1729,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_UnaryOp)
         w_op = unaryop_to_class[self.op - 1]().to_object(space)  # unaryop
-        space.setattr(w_node, space.wrap('op'), w_op)
+        space.setattr(w_node, space.newtext('op'), w_op)
         w_operand = self.operand.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('operand'), w_operand)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('operand'), w_operand)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1775,13 +1775,13 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Lambda)
         w_args = self.args.to_object(space)  # arguments
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1823,15 +1823,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_IfExp)
         w_test = self.test.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('test'), w_test)
+        space.setattr(w_node, space.newtext('test'), w_test)
         w_body = self.body.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('body'), w_body)
+        space.setattr(w_node, space.newtext('body'), w_body)
         w_orelse = self.orelse.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('orelse'), w_orelse)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('orelse'), w_orelse)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1883,17 +1883,17 @@
         else:
             keys_w = [node.to_object(space) for node in self.keys] # expr
         w_keys = space.newlist(keys_w)
-        space.setattr(w_node, space.wrap('keys'), w_keys)
+        space.setattr(w_node, space.newtext('keys'), w_keys)
         if self.values is None:
             values_w = []
         else:
             values_w = [node.to_object(space) for node in self.values] # expr
         w_values = space.newlist(values_w)
-        space.setattr(w_node, space.wrap('values'), w_values)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('values'), w_values)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1935,11 +1935,11 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -1976,17 +1976,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ListComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2027,17 +2027,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_SetComp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2080,19 +2080,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_DictComp)
         w_key = self.key.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('key'), w_key)
+        space.setattr(w_node, space.newtext('key'), w_key)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2137,17 +2137,17 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_GeneratorExp)
         w_elt = self.elt.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('elt'), w_elt)
+        space.setattr(w_node, space.newtext('elt'), w_elt)
         if self.generators is None:
             generators_w = []
         else:
             generators_w = [node.to_object(space) for node in self.generators] # comprehension
         w_generators = space.newlist(generators_w)
-        space.setattr(w_node, space.wrap('generators'), w_generators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('generators'), w_generators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2185,11 +2185,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Yield)
         w_value = self.value.to_object(space) if self.value is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2226,23 +2226,23 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Compare)
         w_left = self.left.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('left'), w_left)
+        space.setattr(w_node, space.newtext('left'), w_left)
         if self.ops is None:
             ops_w = []
         else:
             ops_w = [cmpop_to_class[node - 1]().to_object(space) for node in self.ops] # cmpop
         w_ops = space.newlist(ops_w)
-        space.setattr(w_node, space.wrap('ops'), w_ops)
+        space.setattr(w_node, space.newtext('ops'), w_ops)
         if self.comparators is None:
             comparators_w = []
         else:
             comparators_w = [node.to_object(space) for node in self.comparators] # expr
         w_comparators = space.newlist(comparators_w)
-        space.setattr(w_node, space.wrap('comparators'), w_comparators)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('comparators'), w_comparators)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2296,27 +2296,27 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Call)
         w_func = self.func.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('func'), w_func)
+        space.setattr(w_node, space.newtext('func'), w_func)
         if self.args is None:
             args_w = []
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
+        space.setattr(w_node, space.newtext('args'), w_args)
         if self.keywords is None:
             keywords_w = []
         else:
             keywords_w = [node.to_object(space) for node in self.keywords] # keyword
         w_keywords = space.newlist(keywords_w)
-        space.setattr(w_node, space.wrap('keywords'), w_keywords)
+        space.setattr(w_node, space.newtext('keywords'), w_keywords)
         w_starargs = self.starargs.to_object(space) if self.starargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('starargs'), w_starargs)
+        space.setattr(w_node, space.newtext('starargs'), w_starargs)
         w_kwargs = self.kwargs.to_object(space) if self.kwargs is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('kwargs'), w_kwargs)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('kwargs'), w_kwargs)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2360,11 +2360,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Repr)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2397,11 +2397,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Num)
         w_n = self.n  # object
-        space.setattr(w_node, space.wrap('n'), w_n)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('n'), w_n)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2434,11 +2434,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Str)
         w_s = self.s  # string
-        space.setattr(w_node, space.wrap('s'), w_s)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('s'), w_s)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2474,15 +2474,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Attribute)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_attr = space.wrap(self.attr)  # identifier
-        space.setattr(w_node, space.wrap('attr'), w_attr)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_attr = space.newtext(self.attr)  # identifier
+        space.setattr(w_node, space.newtext('attr'), w_attr)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2495,7 +2495,7 @@
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
-        _attr = space.realstr_w(w_attr)
+        _attr = space.realtext_w(w_attr)
         if _attr is None:
             raise_required_value(space, w_node, 'attr')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -2527,15 +2527,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Subscript)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         w_slice = self.slice.to_object(space)  # slice
-        space.setattr(w_node, space.wrap('slice'), w_slice)
+        space.setattr(w_node, space.newtext('slice'), w_slice)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2576,14 +2576,14 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Name)
-        w_id = space.wrap(self.id)  # identifier
-        space.setattr(w_node, space.wrap('id'), w_id)
+        w_id = space.newtext(self.id)  # identifier
+        space.setattr(w_node, space.newtext('id'), w_id)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2592,7 +2592,7 @@
         w_ctx = get_field(space, w_node, 'ctx', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _id = space.realstr_w(w_id)
+        _id = space.realtext_w(w_id)
         if _id is None:
             raise_required_value(space, w_node, 'id')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -2628,13 +2628,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2678,13 +2678,13 @@
         else:
             elts_w = [node.to_object(space) for node in self.elts] # expr
         w_elts = space.newlist(elts_w)
-        space.setattr(w_node, space.wrap('elts'), w_elts)
+        space.setattr(w_node, space.newtext('elts'), w_elts)
         w_ctx = expr_context_to_class[self.ctx - 1]().to_object(space)  # expr_context
-        space.setattr(w_node, space.wrap('ctx'), w_ctx)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('ctx'), w_ctx)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2720,11 +2720,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Const)
         w_value = self.value  # object
-        space.setattr(w_node, space.wrap('value'), w_value)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('value'), w_value)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -2866,11 +2866,11 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Slice)
         w_lower = self.lower.to_object(space) if self.lower is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('lower'), w_lower)
+        space.setattr(w_node, space.newtext('lower'), w_lower)
         w_upper = self.upper.to_object(space) if self.upper is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('upper'), w_upper)
+        space.setattr(w_node, space.newtext('upper'), w_upper)
         w_step = self.step.to_object(space) if self.step is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('step'), w_step)
+        space.setattr(w_node, space.newtext('step'), w_step)
         return w_node
 
     @staticmethod
@@ -2907,7 +2907,7 @@
         else:
             dims_w = [node.to_object(space) for node in self.dims] # slice
         w_dims = space.newlist(dims_w)
-        space.setattr(w_node, space.wrap('dims'), w_dims)
+        space.setattr(w_node, space.newtext('dims'), w_dims)
         return w_node
 
     @staticmethod
@@ -2935,7 +2935,7 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_Index)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
@@ -3266,15 +3266,15 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_comprehension)
         w_target = self.target.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('target'), w_target)
+        space.setattr(w_node, space.newtext('target'), w_target)
         w_iter = self.iter.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('iter'), w_iter)
+        space.setattr(w_node, space.newtext('iter'), w_iter)
         if self.ifs is None:
             ifs_w = []
         else:
             ifs_w = [node.to_object(space) for node in self.ifs] # expr
         w_ifs = space.newlist(ifs_w)
-        space.setattr(w_node, space.wrap('ifs'), w_ifs)
+        space.setattr(w_node, space.newtext('ifs'), w_ifs)
         return w_node
 
     @staticmethod
@@ -3334,19 +3334,19 @@
     def to_object(self, space):
         w_node = space.call_function(get(space).w_ExceptHandler)
         w_type = self.type.to_object(space) if self.type is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('type'), w_type)
+        space.setattr(w_node, space.newtext('type'), w_type)
         w_name = self.name.to_object(space) if self.name is not None else space.w_None  # expr
-        space.setattr(w_node, space.wrap('name'), w_name)
+        space.setattr(w_node, space.newtext('name'), w_name)
         if self.body is None:
             body_w = []
         else:
             body_w = [node.to_object(space) for node in self.body] # stmt
         w_body = space.newlist(body_w)
-        space.setattr(w_node, space.wrap('body'), w_body)
-        w_lineno = space.wrap(self.lineno)  # int
-        space.setattr(w_node, space.wrap('lineno'), w_lineno)
-        w_col_offset = space.wrap(self.col_offset)  # int
-        space.setattr(w_node, space.wrap('col_offset'), w_col_offset)
+        space.setattr(w_node, space.newtext('body'), w_body)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
         return w_node
 
     @staticmethod
@@ -3394,17 +3394,17 @@
         else:
             args_w = [node.to_object(space) for node in self.args] # expr
         w_args = space.newlist(args_w)
-        space.setattr(w_node, space.wrap('args'), w_args)
-        w_vararg = space.wrap(self.vararg)  # identifier
-        space.setattr(w_node, space.wrap('vararg'), w_vararg)
-        w_kwarg = space.wrap(self.kwarg)  # identifier
-        space.setattr(w_node, space.wrap('kwarg'), w_kwarg)
+        space.setattr(w_node, space.newtext('args'), w_args)
+        w_vararg = space.newtext_or_none(self.vararg)  # identifier
+        space.setattr(w_node, space.newtext('vararg'), w_vararg)
+        w_kwarg = space.newtext_or_none(self.kwarg)  # identifier
+        space.setattr(w_node, space.newtext('kwarg'), w_kwarg)
         if self.defaults is None:
             defaults_w = []
         else:
             defaults_w = [node.to_object(space) for node in self.defaults] # expr
         w_defaults = space.newlist(defaults_w)
-        space.setattr(w_node, space.wrap('defaults'), w_defaults)
+        space.setattr(w_node, space.newtext('defaults'), w_defaults)
         return w_node
 
     @staticmethod
@@ -3415,8 +3415,8 @@
         w_defaults = get_field(space, w_node, 'defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [expr.from_object(space, w_item) for w_item in args_w]
-        _vararg = space.str_or_None_w(w_vararg)
-        _kwarg = space.str_or_None_w(w_kwarg)
+        _vararg = space.realtext_w(w_vararg) if not space.is_none(w_vararg) else None
+        _kwarg = space.realtext_w(w_kwarg) if not space.is_none(w_kwarg) else None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwarg, _defaults)
@@ -3438,17 +3438,17 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_keyword)
-        w_arg = space.wrap(self.arg)  # identifier
-        space.setattr(w_node, space.wrap('arg'), w_arg)
+        w_arg = space.newtext(self.arg)  # identifier
+        space.setattr(w_node, space.newtext('arg'), w_arg)
         w_value = self.value.to_object(space)  # expr
-        space.setattr(w_node, space.wrap('value'), w_value)
+        space.setattr(w_node, space.newtext('value'), w_value)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', False)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.realstr_w(w_arg)
+        _arg = space.realtext_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
         _value = expr.from_object(space, w_value)
@@ -3472,20 +3472,20 @@
 
     def to_object(self, space):
         w_node = space.call_function(get(space).w_alias)
-        w_name = space.wrap(self.name)  # identifier
-        space.setattr(w_node, space.wrap('name'), w_name)
-        w_asname = space.wrap(self.asname)  # identifier
-        space.setattr(w_node, space.wrap('asname'), w_asname)
+        w_name = space.newtext(self.name)  # identifier
+        space.setattr(w_node, space.newtext('name'), w_name)
+        w_asname = space.newtext_or_none(self.asname)  # identifier
+        space.setattr(w_node, space.newtext('asname'), w_asname)
         return w_node
 
     @staticmethod
     def from_object(space, w_node):
         w_name = get_field(space, w_node, 'name', False)
         w_asname = get_field(space, w_node, 'asname', True)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.str_or_None_w(w_asname)
+        _asname = space.realtext_w(w_asname) if not space.is_none(w_asname) else None
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1070,8 +1070,8 @@
                 raw = "0" + raw
         if negative:
             raw = "-" + raw
-        w_num_str = self.space.wrap(raw)
-        w_base = self.space.wrap(base)
+        w_num_str = self.space.newtext(raw)
+        w_base = self.space.newint(base)
         if raw[-1] in "lL":
             tp = self.space.w_long
             return self.space.call_function(tp, w_num_str, w_base)
@@ -1105,12 +1105,12 @@
                     raise
                 # UnicodeError in literal: turn into SyntaxError
                 e.normalize_exception(space)
-                errmsg = space.str_w(space.str(e.get_w_value(space)))
+                errmsg = space.text_w(space.str(e.get_w_value(space)))
                 raise self.error('(unicode error) %s' % errmsg, atom_node)
             # This implements implicit string concatenation.
             if len(sub_strings_w) > 1:
                 w_sub_strings = space.newlist(sub_strings_w)
-                w_join = space.getattr(space.wrap(""), space.wrap("join"))
+                w_join = space.getattr(space.newtext(""), space.newtext("join"))
                 final_string = space.call_function(w_join, w_sub_strings)
             else:
                 final_string = sub_strings_w[0]
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -322,7 +322,7 @@
     def visit_ClassDef(self, cls):
         self.update_position(cls.lineno, True)
         self.visit_sequence(cls.decorator_list)
-        self.load_const(self.space.wrap(cls.name))
+        self.load_const(self.space.newtext(cls.name))
         self.visit_sequence(cls.bases)
         bases_count = len(cls.bases) if cls.bases is not None else 0
         self.emit_op_arg(ops.BUILD_TUPLE, bases_count)
@@ -611,7 +611,7 @@
                 level = 0
             else:
                 level = -1
-            self.load_const(self.space.wrap(level))
+            self.load_const(self.space.newint(level))
             self.load_const(self.space.w_None)
             self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name)
             # If there's no asname then we store the root module.  If there is
@@ -654,12 +654,12 @@
             level = -1
         else:
             level = imp.level
-        self.load_const(space.wrap(level))
+        self.load_const(space.newint(level))
         names_w = [None]*len(imp.names)
         for i in range(len(imp.names)):
             alias = imp.names[i]
             assert isinstance(alias, ast.alias)
-            names_w[i] = space.wrap(alias.name)
+            names_w[i] = space.newtext(alias.name)
         self.load_const(space.newtuple(names_w))
         if imp.module:
             mod_name = imp.module
@@ -944,7 +944,7 @@
         self.name_op(name.id, name.ctx)
 
     def visit_keyword(self, keyword):
-        self.load_const(self.space.wrap(keyword.arg))
+        self.load_const(self.space.newtext(keyword.arg))
         keyword.value.walkabout(self)
 
     def visit_Call(self, call):
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -19,10 +19,10 @@
 
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
-    w_msg = space.wrap(msg)
-    w_filename = space.wrap(fn)
-    w_lineno = space.wrap(lineno)
-    w_offset = space.wrap(offset)
+    w_msg = space.newtext(msg)
+    w_filename = space.newtext(fn)
+    w_lineno = space.newint(lineno)
+    w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
 
 
@@ -110,9 +110,9 @@
 
 def intern_if_common_string(space, w_const):
     # only intern identifier-like strings
-    if not space.is_w(space.type(w_const), space.w_str):
+    if not space.is_w(space.type(w_const), space.w_text):
         return w_const
-    for c in space.str_w(w_const):
+    for c in space.text_w(w_const):
         if not (c.isalnum() or c == '_'):
             return w_const
     return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -120,7 +120,7 @@
     return space.pow(w_left, w_right, space.w_None)
 
 def _fold_not(space, operand):
-    return space.wrap(not space.is_true(operand))
+    return space.newbool(not space.is_true(operand))
 
 
 binary_folders = {
@@ -216,7 +216,7 @@
                         break
                 else:
                     raise AssertionError("unknown unary operation")
-                w_minint = self.space.wrap(-sys.maxint - 1)
+                w_minint = self.space.newint(-sys.maxint - 1)
                 # This makes sure the result is an integer.
                 if self.space.eq_w(w_minint, w_const):
                     w_const = w_minint
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -127,8 +127,14 @@
             return "%s_to_class[%s - 1]().to_object(space)" % (field.type, value)
         elif field.type in ("object", "singleton", "string", "bytes"):
             return value
-        elif field.type in ("identifier", "int", "bool"):
-            return "space.wrap(%s)" % (value,)
+        elif field.type == "bool":
+            return "space.newbool(%s)" % (value,)
+        elif field.type == "int":
+            return "space.newint(%s)" % (value,)
+        elif field.type == "identifier":
+            if field.opt:
+                return "space.newtext_or_none(%s)" % (value,)
+            return "space.newtext(%s)" % (value,)
         else:
             wrapper = "%s.to_object(space)" % (value,)
             if field.opt:
@@ -144,8 +150,9 @@
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
-                return "space.str_or_None_w(%s)" % (value,)
-            return "space.realstr_w(%s)" % (value,)
+                return ("space.realtext_w(%s) if not space.is_none(%s) "
+                        "else None" % (value, value))
+            return "space.realtext_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
         elif field.type in ("bool",):
@@ -194,7 +201,7 @@
             wrapping_code = self.get_field_converter(field)
             for line in wrapping_code:
                 self.emit(line, 2)
-            self.emit("space.setattr(w_node, space.wrap(%r), w_%s)" % (
+            self.emit("space.setattr(w_node, space.newtext(%r), w_%s)" % (
                     str(field.name), field.name), 2)
         self.emit("return w_node", 2)
         self.emit("")
@@ -402,7 +409,7 @@
                 "field %s is required for %T", name, w_obj)
 
 def check_string(space, w_obj):
-    if not (space.isinstance_w(w_obj, space.w_str) or
+    if not (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_unicode)):
         raise oefmt(space.w_TypeError,
                     "AST string must be of type str or unicode")
@@ -437,10 +444,10 @@
     "Hack around the fact we can't store tuples on a TypeDef."
 
     def __init__(self, fields):
-        self.fields = fields
+        assert fields == []
 
-    def __spacebind__(self, space):
-        return space.newtuple([space.wrap(field) for field in self.fields])
+    def spacebind(self, space):
+        return space.newtuple([])
 
 
 class W_AST(W_Root):
@@ -456,14 +463,14 @@
         if w_dict is None:
             w_dict = space.newdict()
         w_type = space.type(self)
-        w_fields = space.getattr(w_type, space.wrap("_fields"))
+        w_fields = space.getattr(w_type, space.newtext("_fields"))
         for w_name in space.fixedview(w_fields):
             try:
                 space.setitem(w_dict, w_name,
                           space.getattr(self, w_name))
             except OperationError:
                 pass
-        w_attrs = space.findattr(w_type, space.wrap("_attributes"))
+        w_attrs = space.findattr(w_type, space.newtext("_attributes"))
         if w_attrs:
             for w_name in space.fixedview(w_attrs):
                 try:
@@ -482,12 +489,12 @@
 
 def W_AST_new(space, w_type, __args__):
     node = space.allocate_instance(W_AST, w_type)
-    return space.wrap(node)
+    return node
 
 def W_AST_init(space, w_self, __args__):
     args_w, kwargs_w = __args__.unpack()
     fields_w = space.fixedview(space.getattr(space.type(w_self),
-                               space.wrap("_fields")))
+                               space.newtext("_fields")))
     num_fields = len(fields_w) if fields_w else 0
     if args_w and len(args_w) != num_fields:
         if num_fields == 0:
@@ -503,7 +510,7 @@
         for i, w_field in enumerate(fields_w):
             space.setattr(w_self, w_field, args_w[i])
     for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
+        space.setattr(w_self, space.newtext(field), w_value)
 
 
 W_AST.typedef = typedef.TypeDef("_ast.AST",
@@ -532,16 +539,16 @@
     def make_new_type(self, space, name, base, fields, attributes):
         w_base = getattr(self, 'w_%s' % base)
         w_dict = space.newdict()
-        space.setitem_str(w_dict, '__module__', space.wrap('_ast'))
+        space.setitem_str(w_dict, '__module__', space.newtext('_ast'))
         if fields is not None:
             space.setitem_str(w_dict, "_fields",
-                              space.newtuple([space.wrap(f) for f in fields]))
+                              space.newtuple([space.newtext(f) for f in fields]))
         if attributes is not None:
             space.setitem_str(w_dict, "_attributes",
-                              space.newtuple([space.wrap(a) for a in attributes]))
+                              space.newtuple([space.newtext(a) for a in attributes]))
         w_type = space.call_function(
             space.w_type,
-            space.wrap(name), space.newtuple([w_base]), w_dict)

From pypy.commits at gmail.com  Wed Feb 22 16:13:59 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 22 Feb 2017 13:13:59 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-3: add some comments,
 some nicer var names, a tiny bit of cleanup
Message-ID: <58adff17.18532e0a.7645f.e953@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-3
Changeset: r90315:c8e2eb2e3e18
Date: 2017-02-22 22:13 +0100
http://bitbucket.org/pypy/pypy/changeset/c8e2eb2e3e18/

Log:	add some comments, some nicer var names, a tiny bit of cleanup

diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
--- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -3,8 +3,6 @@
 
 from rpython.jit.metainterp import resumecode
 
-# XXX at the moment this is all quite ad-hoc. Could be delegated to the
-# different optimization passes
 
 # adds the following sections at the end of the resume code:
 #
@@ -22,9 +20,11 @@
 #
 # ----
 
+
+# maybe should be delegated to the optimization classes?
+
 def tag_box(box, liveboxes_from_env, memo):
     from rpython.jit.metainterp.history import Const
-    # XXX bit of code duplication (but it's a subset)
     if isinstance(box, Const):
         return memo.getconst(box)
     else:
@@ -35,6 +35,8 @@
     from rpython.jit.metainterp.resume import NULLREF, TAG_CONST_OFFSET, tagged_eq
     from rpython.jit.metainterp.history import ConstInt
     num, tag = untag(tagged)
+    # NB: the TAGVIRTUAL case can't happen here, because this code runs after
+    # virtuals are already forced again
     if tag == TAGCONST:
         if tagged_eq(tagged, NULLREF):
             box = cpu.ts.CONST_NULL
@@ -49,13 +51,17 @@
     return box
 
 def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo):
-    liveboxes_set = {}
+    available_boxes = {}
     for box in liveboxes:
         if box is not None and box in liveboxes_from_env:
-            liveboxes_set[box] = None
+            available_boxes[box] = None
     metainterp_sd = optimizer.metainterp_sd
 
-    # class knowledge
+    # class knowledge is stored as bits, true meaning the class is known, false
+    # means unknown. on deserializing we look at the bits, and read the runtime
+    # class for the known classes (which has to be the same in the bridge) and
+    # mark that as known. this works for guard_class too: the class is only
+    # known *after* the guard
     bitfield = 0
     shifts = 0
     for box in liveboxes:
@@ -72,9 +78,11 @@
     if shifts:
         numb_state.append_int(bitfield << (6 - shifts))
 
-    # heap knowledge
+    # heap knowledge: we store triples of known heap fields in non-virtual
+    # structs
+    # XXX could be extended to arrays
     if optimizer.optheap:
-        triples = optimizer.optheap.serialize_optheap(liveboxes_set)
+        triples = optimizer.optheap.serialize_optheap(available_boxes)
         # can only encode descrs that have a known index into
         # metainterp_sd.all_descrs
         triples = [triple for triple in triples if triple[1].descr_index != -1]
@@ -112,6 +120,8 @@
             optimizer.make_constant_class(box, cls)
 
     # heap knowledge
+    if not optimizer.optheap:
+        return
     length = reader.next_item()
     result = []
     for i in range(length):
@@ -122,5 +132,4 @@
         tagged = reader.next_item()
         box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
         result.append((box1, descr, box2))
-    if optimizer.optheap:
-        optimizer.optheap.deserialize_optheap(result)
+    optimizer.optheap.deserialize_optheap(result)
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -694,21 +694,20 @@
         self._seen_guard_not_invalidated = True
         return self.emit(op)
 
-    def serialize_optheap(self, liveboxes_set):
-        # XXX wrong complexity?
+    def serialize_optheap(self, available_boxes):
         result = []
         for descr, cf in self.cached_fields.iteritems():
             if cf._lazy_set:
                 continue # XXX safe default for now
             parent_descr = descr.get_parent_descr()
             if not parent_descr.is_object():
-                continue
+                continue # XXX could be extended to non-instance objects
             for i, box1 in enumerate(cf.cached_structs):
-                if box1 not in liveboxes_set:
+                if box1 not in available_boxes:
                     continue
                 structinfo = cf.cached_infos[i]
                 box2 = structinfo.getfield(descr).get_box_replacement()
-                if isinstance(box2, Const) or box2 in liveboxes_set:
+                if isinstance(box2, Const) or box2 in available_boxes:
                     result.append((box1, descr, box2))
         return result
 
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -121,12 +121,12 @@
         self.items_read = 0 # number of items read
 
     def next_item(self):
-        result, self.cur_pos =  numb_next_item(self.code, self.cur_pos)
+        result, self.cur_pos = numb_next_item(self.code, self.cur_pos)
         self.items_read += 1
         return result
 
     def peek(self):
-        result, _ =  numb_next_item(self.code, self.cur_pos)
+        result, _ = numb_next_item(self.code, self.cur_pos)
         return result
 
     def jump(self, size):
diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py
--- a/rpython/jit/metainterp/test/test_bridgeopt.py
+++ b/rpython/jit/metainterp/test/test_bridgeopt.py
@@ -97,7 +97,6 @@
 
     serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
 
-    #assert numb_state.current[:numb_state._pos] == [1, 0b0100000, 0]
     assert len(numb_state.create_numbering().code) == 2 + math.ceil(len(refboxes) / 6.0)
 
     dct = {box: cls

From pypy.commits at gmail.com  Wed Feb 22 23:25:32 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 22 Feb 2017 20:25:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix handling of C-created 0-D buffers
Message-ID: <58ae643c.97212e0a.a38fd.0a36@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90316:91d10f04115f
Date: 2017-02-23 04:24 +0000
http://bitbucket.org/pypy/pypy/changeset/91d10f04115f/

Log:	Fix handling of C-created 0-D buffers

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -82,7 +82,7 @@
     # Allow subclassing W_MemeoryView
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_MemoryView, w_type)
-    w_obj.__init__(buf)
+    w_obj.__init__(buf, itemsize=buf.itemsize)
     track_reference(space, obj, w_obj)
     return w_obj
 
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -320,19 +320,30 @@
         self.space = space
         self.ptr = ptr
         self.size = size
-        self.w_obj = w_obj # kept alive
+        self.w_obj = w_obj  # kept alive
         self.pyobj = make_ref(space, w_obj)
         self.format = format
-        if shape is None:
-            self.shape = [size]
-        else:
-            self.shape = shape
-        if strides is None:
-            self.strides = [1]
-        else:
-            self.strides = strides
         self.ndim = ndim
         self.itemsize = itemsize
+
+        # cf. Objects/memoryobject.c:init_shape_strides()
+        if ndim == 0:
+            self.shape = []
+            self.strides = []
+        elif ndim == 1:
+            if shape is None:
+                self.shape = [size // itemsize]
+            else:
+                self.shape = shape
+            if strides is None:
+                self.strides = [itemsize]
+            else:
+                self.strides = strides
+        else:
+            assert len(shape) == ndim
+            self.shape = shape
+            # XXX: missing init_strides_from_shape
+            self.strides = strides
         self.readonly = readonly
         self.releasebufferproc = releasebufferproc
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -73,6 +73,28 @@
         assert b"hello, world." == result
         del result
 
+    def test_0d(self):
+        module = self.import_extension('foo', [
+            ("create_view", "METH_VARARGS",
+             """
+             /* Create an approximation of the buffer for a 0d ndarray */
+             Py_buffer buf;
+             PyObject *str = PyBytes_FromString("hello, world.");
+             buf.buf = PyBytes_AsString(str);
+             buf.obj = str;
+             buf.readonly = 1;
+             buf.len = 13;
+             buf.itemsize = 13;
+             buf.ndim = 0;
+             buf.shape = NULL;
+             PyObject* ret = PyMemoryView_FromBuffer(&buf);
+             return ret;
+            """)])
+        result = module.create_view()
+        assert result.shape == ()
+        assert result.itemsize == 13
+        assert result.tobytes() == b'hello, world.'
+
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol_app(self):
         module = self.import_module(name='buffer_test')

From pypy.commits at gmail.com  Thu Feb 23 04:54:39 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 23 Feb 2017 01:54:39 -0800 (PST)
Subject: [pypy-commit] stmgc default: add debugging help to README
Message-ID: <58aeb15f.01132e0a.ce08c.02d7@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r2013:995fb3d6ad33
Date: 2017-02-23 10:54 +0100
http://bitbucket.org/pypy/stmgc/changeset/995fb3d6ad33/

Log:	add debugging help to README

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -101,6 +101,58 @@
  example: `make build-demo_random2`
  2. then run `./build-demo_random2`
  
+ 
+### Debugging
+
+GDB works fine for debugging programs with the STMGC library. However, you have
+to tell GDB to ignore `SIGSEGV` by default. A `.gdbinit` could look like this:
+
+    handle SIGSEGV nostop pass noprint
+
+    define sigon
+        handle SIGSEGV stop nopass print
+    end
+
+    define sigoff
+        handle SIGSEGV nostop pass noprint
+    end
+
+    define lon
+        set scheduler-locking on
+    end
+    define loff
+        set scheduler-locking off
+    end
+    
+    # run until crash
+    define runloop
+        set pagination off
+        p $_exitcode = 0
+        while $_exitcode == 0
+            p $_exitcode = -1
+            r
+        end
+        set pagination on
+    end
+
+
+The commands `sigon` and `sigoff` enable and disable `SIGSEGV`-handling. `lon`
+and `loff` enables and disables stopping of other threads while stepping through
+one of them. After reaching a breakpoint in GDB, I usually run `sigon` and `lon`
+to enable GDB to handle real `SIGSEGV` (e.g., while printing) and to stop other
+threads.
+
+`runloop` re-runs a program until there is a crash (useful for reproducing rare
+race conditions).
+
+Furthermore, there are some useful GDB extensions under `/c7/gdb/gdb_stm.py`
+that allow for inspecting segment-local pointers. To enable them, add the
+following line to your `.gdbinit`:
+
+    python exec(open('PATH-TO-STMGC/c7/gdb/gdb_stm.py').read())
+    
+
+
 
 ## Building PyPy-STM
 

From pypy.commits at gmail.com  Thu Feb 23 05:53:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 02:53:46 -0800 (PST)
Subject: [pypy-commit] pypy default: silence a warning
Message-ID: <58aebf3a.5c582e0a.c975c.fcbf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90317:9924a09b1213
Date: 2017-02-23 11:53 +0100
http://bitbucket.org/pypy/pypy/changeset/9924a09b1213/

Log:	silence a warning

diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -178,7 +178,6 @@
             ("bounce", "METH_NOARGS",
             """
             PyGILState_STATE gilstate;
-            PyThreadState *tstate;
             PyObject *dict;
 
             if (PyEval_ThreadsInitialized() == 0)

From pypy.commits at gmail.com  Thu Feb 23 05:54:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 02:54:48 -0800 (PST)
Subject: [pypy-commit] pypy default: don't document branch
Message-ID: <58aebf78.95012e0a.99630.036e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90318:023acd401dbd
Date: 2017-02-23 11:54 +0100
http://bitbucket.org/pypy/pypy/changeset/023acd401dbd/

Log:	don't document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -151,6 +151,8 @@
 not on PyPy 3.x.  The latter is used to get an app-level unicode string
 by decoding the RPython string, assumed to be utf-8.
 
+.. branch: space-wrap
+
 .. branch: fix_bool_restype
 
 Fix for ``ctypes.c_bool``-returning ctypes functions

From pypy.commits at gmail.com  Thu Feb 23 06:33:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 03:33:34 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: Similar to gc-forkfriendly,
 but try to move just the GCFLAG_VISITED offline, 
Message-ID: <58aec88e.97e1190a.283cd.4f7a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90319:6e3d6e6d593e
Date: 2017-02-23 10:23 +0100
http://bitbucket.org/pypy/pypy/changeset/6e3d6e6d593e/

Log:	Similar to gc-forkfriendly, but try to move just the GCFLAG_VISITED
	offline, and not for all objects but only the ones that live in
	minimarkpage.


From pypy.commits at gmail.com  Thu Feb 23 06:33:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 03:33:39 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58aec893.1ce5190a.dd0bb.5c3a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90321:1c47dc995f88
Date: 2017-02-23 12:32 +0100
http://bitbucket.org/pypy/pypy/changeset/1c47dc995f88/

Log:	merge heads

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -5,7 +5,7 @@
 from rpython.rlib.unicodedata import unicodedb
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import jit
+from rpython.rlib import jit, nonconst
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -373,7 +373,12 @@
                             pos += 1
                             _encodeUCS4(result, ch3)
                             continue
-                    if not allow_surrogates:
+                    # note: if the program only ever calls this with
+                    # allow_surrogates=True, then we'll never annotate
+                    # the following block of code, and errorhandler()
+                    # will never be called.  This causes RPython
+                    # problems.  Avoid it with the nonconst hack.
+                    if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
                                                    s, pos-1, pos)

From pypy.commits at gmail.com  Thu Feb 23 06:33:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 03:33:36 -0800 (PST)
Subject: [pypy-commit] pypy default: attempt n+1 at a fix
Message-ID: <58aec890.53be190a.58996.5b5b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90320:208633bf8fea
Date: 2017-02-23 11:50 +0100
http://bitbucket.org/pypy/pypy/changeset/208633bf8fea/

Log:	attempt n+1 at a fix

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -5,7 +5,7 @@
 from rpython.rlib.unicodedata import unicodedb
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import jit
+from rpython.rlib import jit, nonconst
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -373,7 +373,12 @@
                             pos += 1
                             _encodeUCS4(result, ch3)
                             continue
-                    if not allow_surrogates:
+                    # note: if the program only ever calls this with
+                    # allow_surrogates=True, then we'll never annotate
+                    # the following block of code, and errorhandler()
+                    # will never be called.  This causes RPython
+                    # problems.  Avoid it with the nonconst hack.
+                    if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
                                                    s, pos-1, pos)

From pypy.commits at gmail.com  Thu Feb 23 08:15:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 05:15:57 -0800 (PST)
Subject: [pypy-commit] pypy reverse-debugger: Give id()s that are generally
 multiple of 8. Helps running rpython tests.
Message-ID: <58aee08d.04162e0a.21a12.1510@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r90322:ff1be8a8ca2a
Date: 2017-02-23 14:15 +0100
http://bitbucket.org/pypy/pypy/changeset/ff1be8a8ca2a/

Log:	Give id()s that are generally multiple of 8. Helps running rpython
	tests.

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -452,7 +452,14 @@
                 warning_printed = 1;
             }
         }
-        return (Signed)(obj->h_uid);
+        /* on 64-bit, the range is large enough anyway, so shift 3 times
+           to get a result that is a multiple of 8.  The Boehm translation
+           will additionally return the 'int_invert' of that.  So
+           we add 7 to make the user-level id() be a multiple of 8. */
+        if (sizeof(obj->h_uid) == sizeof(Signed))
+            return (obj->h_uid << 3) + 7;
+        else
+            return (Signed)(obj->h_uid);
     }
 }
 

From pypy.commits at gmail.com  Thu Feb 23 08:46:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 05:46:21 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: Tweak an existing gc flag so
 that it can be used as an indicator for
Message-ID: <58aee7ad.97e1190a.283cd.55aa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90323:6aee6fc9c814
Date: 2017-02-23 14:42 +0100
http://bitbucket.org/pypy/pypy/changeset/6aee6fc9c814/

Log:	Tweak an existing gc flag so that it can be used as an indicator for
	whether an old object is inside a minimarkpage-allocated arena or
	not

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -71,7 +71,7 @@
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint
 from rpython.rlib.rarithmetic import LONG_BIT_SHIFT
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.memory.gc.minimarkpage import out_of_memory
 
 #
@@ -135,9 +135,13 @@
 # note that GCFLAG_CARDS_SET is the most significant bit of a byte:
 # this is required for the JIT (x86)
 
-# The following flag is set on surviving raw-malloced young objects during
-# a minor collection.
-GCFLAG_VISITED_RMY   = first_gcflag << 8
+# The following flag is set on all *old* objects that are not allocated
+# with the normal minimarkpage mechanism.  During minor collections, it
+# is used to track surviving raw-malloced young objects (they don't have
+# the flag to start with, but it gets added if they survive).  Once the
+# minor collection is complete, all objects have the flag iff they are
+# not located inside the minimarkpage.
+GCFLAG_OLD_OUTSIDE_MINIMARKPAGE = first_gcflag << 8
 
 # The following flag is set on nursery objects to keep them in the nursery.
 # This means that a young object with this flag is not moved out
@@ -1002,7 +1006,8 @@
                 self.young_rawmalloced_objects.add(result + size_gc_header)
             else:
                 self.old_rawmalloced_objects.append(result + size_gc_header)
-                extra_flags |= GCFLAG_TRACK_YOUNG_PTRS
+                extra_flags |= (GCFLAG_TRACK_YOUNG_PTRS |
+                                GCFLAG_OLD_OUTSIDE_MINIMARKPAGE)
         #
         # Common code to fill the header and length of the object.
         self.init_gc_object(result, typeid, extra_flags)
@@ -1126,7 +1131,8 @@
     def init_gc_object_immortal(self, addr, typeid16, flags=0):
         # For prebuilt GC objects, the flags must contain
         # GCFLAG_NO_xxx_PTRS, at least initially.
-        flags |= GCFLAG_NO_HEAP_PTRS | GCFLAG_TRACK_YOUNG_PTRS
+        flags |= (GCFLAG_NO_HEAP_PTRS | GCFLAG_TRACK_YOUNG_PTRS
+                  | GCFLAG_OLD_OUTSIDE_MINIMARKPAGE)
         self.init_gc_object(addr, typeid16, flags)
 
     def is_in_nursery(self, addr):
@@ -1241,13 +1247,21 @@
         if not self._is_pinned(obj):
             ll_assert(not self.is_in_nursery(obj),
                       "object in nursery after collection")
-            ll_assert(self.header(obj).tid & GCFLAG_VISITED_RMY == 0,
-                      "GCFLAG_VISITED_RMY after collection")
+            if not we_are_translated():
+                size_gc_header = self.gcheaderbuilder.size_gc_header
+                if self.ac._is_inside_minimarkpage(obj - size_gc_header):
+                    expected_flag = 0
+                else:
+                    expected_flag = GCFLAG_OLD_OUTSIDE_MINIMARKPAGE
+                assert ((self.header(obj).tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE)
+                        == expected_flag)
             ll_assert(self.header(obj).tid & GCFLAG_PINNED == 0,
                       "GCFLAG_PINNED outside the nursery after collection")
         else:
             ll_assert(self.is_in_nursery(obj),
                       "pinned object not in nursery")
+            ll_assert((self.header(obj).tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE)
+                      != 0, "pinned object must be OLD_OUTSIDE_MINIMARKPAGE")
 
         if self.gc_state == STATE_SCANNING:
             self._debug_check_object_scanning(obj)
@@ -1660,7 +1674,8 @@
         #
         # First, find the roots that point to young objects.  All nursery
         # objects found are copied out of the nursery, and the occasional
-        # young raw-malloced object is flagged with GCFLAG_VISITED_RMY.
+        # young raw-malloced object is flagged with
+        # GCFLAG_OLD_OUTSIDE_MINIMARKPAGE.
         # Note that during this step, we ignore references to further
         # young objects; only objects directly referenced by roots
         # are copied out or flagged.  They are also added to the list
@@ -1702,7 +1717,7 @@
             # All nursery objects they reference are copied out of the
             # nursery, and again added to 'old_objects_pointing_to_young'.
             # All young raw-malloced object found are flagged
-            # GCFLAG_VISITED_RMY.
+            # GCFLAG_OLD_OUTSIDE_MINIMARKPAGE.
             # We proceed until 'old_objects_pointing_to_young' is empty.
             self.collect_oldrefs_to_nursery()
             #
@@ -1979,7 +1994,8 @@
         #print '_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj)
         #
         # If 'obj' is not in the nursery, nothing to change -- expect
-        # that we must set GCFLAG_VISITED_RMY on young raw-malloced objects.
+        # that we must set GCFLAG_OLD_OUTSIDE_MINIMARKPAGE on young
+        # raw-malloced objects.
         if not self.is_in_nursery(obj):
             # cache usage trade-off: I think that it is a better idea to
             # check if 'obj' is in young_rawmalloced_objects with an access
@@ -1993,6 +2009,7 @@
         # copy the contents of the object? usually yes, but not for some
         # shadow objects
         copy = True
+        extra_flag = 0
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
@@ -2003,7 +2020,7 @@
             # into a new nonmovable location.
             totalsize = size_gc_header + self.get_size(obj)
             self.nursery_surviving_size += raw_malloc_usage(totalsize)
-            newhdr = self._malloc_out_of_nursery(totalsize)
+            newhdr, extra_flag = self._malloc_out_of_nursery(totalsize)
             #
         elif self.is_forwarded(obj):
             #
@@ -2032,7 +2049,7 @@
             if hdr.tid & GCFLAG_VISITED:
                 return
             #
-            hdr.tid |= GCFLAG_VISITED
+            hdr.tid |= GCFLAG_VISITED | GCFLAG_OLD_OUTSIDE_MINIMARKPAGE
             #
             self.surviving_pinned_objects.append(
                 llarena.getfakearenaaddress(obj - size_gc_header))
@@ -2055,11 +2072,15 @@
             #
             totalsize = size_gc_header + self.get_size(obj)
             self.nursery_surviving_size += raw_malloc_usage(totalsize)
+            extra_flag = (
+                self.header(newobj).tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
         if copy:
+            tid = self.header(obj).tid &~ GCFLAG_OLD_OUTSIDE_MINIMARKPAGE
             llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+            self.header(newhdr + size_gc_header).tid = tid | extra_flag
         #
         # Set the old object's tid to -42 (containing all flags) and
         # replace the old object's content with the target address.
@@ -2090,15 +2111,16 @@
     def _visit_young_rawmalloced_object(self, obj):
         # 'obj' points to a young, raw-malloced object.
         # Any young rawmalloced object never seen by the code here
-        # will end up without GCFLAG_VISITED_RMY, and be freed at the
+        # will end up without GCFLAG_OLD_OUTSIDE_MINIMARKPAGE,
+        # and be freed at the
         # end of the current minor collection.  Note that there was
         # a bug in which dying young arrays with card marks would
         # still be scanned before being freed, keeping a lot of
         # objects unnecessarily alive.
         hdr = self.header(obj)
-        if hdr.tid & GCFLAG_VISITED_RMY:
+        if hdr.tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE:
             return
-        hdr.tid |= GCFLAG_VISITED_RMY
+        hdr.tid |= GCFLAG_OLD_OUTSIDE_MINIMARKPAGE
         #
         # Accounting
         size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -2126,10 +2148,11 @@
         'totalsize' that lives so far in the nursery."""
         if raw_malloc_usage(totalsize) <= self.small_request_threshold:
             # most common path
-            return self.ac.malloc(totalsize)
+            return (self.ac.malloc(totalsize), 0)
         else:
             # for nursery objects that are not small
-            return self._malloc_out_of_nursery_nonsmall(totalsize)
+            return (self._malloc_out_of_nursery_nonsmall(totalsize),
+                    GCFLAG_OLD_OUTSIDE_MINIMARKPAGE)
     _malloc_out_of_nursery._always_inline_ = True
 
     def _malloc_out_of_nursery_nonsmall(self, totalsize):
@@ -2154,9 +2177,10 @@
         self.young_rawmalloced_objects = self.null_address_dict()
 
     def _free_young_rawmalloced_obj(self, obj, ignored1, ignored2):
-        # If 'obj' has GCFLAG_VISITED_RMY, it was seen by _trace_drag_out
-        # and survives.  Otherwise, it dies.
-        self.free_rawmalloced_object_if_unvisited(obj, GCFLAG_VISITED_RMY)
+        # If 'obj' has GCFLAG_OLD_OUTSIDE_MINIMARKPAGE, it was seen by
+        # _trace_drag_out and survives.  Otherwise, it dies.
+        self.free_rawmalloced_object_if_unvisited(obj,
+            GCFLAG_OLD_OUTSIDE_MINIMARKPAGE, remove_flag=False)
 
     def remove_young_arrays_from_old_objects_pointing_to_young(self):
         old = self.old_objects_pointing_to_young
@@ -2434,9 +2458,11 @@
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
 
-    def free_rawmalloced_object_if_unvisited(self, obj, check_flag):
+    def free_rawmalloced_object_if_unvisited(self, obj, check_flag,
+                                             remove_flag=True):
         if self.header(obj).tid & check_flag:
-            self.header(obj).tid &= ~check_flag   # survives
+            if remove_flag:
+                self.header(obj).tid &= ~check_flag   # survives
             self.old_rawmalloced_objects.append(obj)
         else:
             size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -2590,8 +2616,9 @@
     def _allocate_shadow(self, obj, copy=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         size = self.get_size(obj)
-        shadowhdr = self._malloc_out_of_nursery(size_gc_header +
-                                                size)
+        shadowhdr, shadowflag = self._malloc_out_of_nursery(size_gc_header +
+                                                            size)
+
         # Initialize the shadow enough to be considered a
         # valid gc object.  If the original object stays
         # alive at the next minor collection, it will anyway
@@ -2615,6 +2642,9 @@
             totalsize = size_gc_header + self.get_size(obj)
             llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize)
 
+        if shadowflag != 0:
+            self.header(shadow).tid |= shadowflag
+
         return shadow
 
     def _find_shadow(self, obj):
@@ -2847,7 +2877,8 @@
             elif (bool(self.young_rawmalloced_objects) and
                   self.young_rawmalloced_objects.contains(pointing_to)):
                 # young weakref to a young raw-malloced object
-                if self.header(pointing_to).tid & GCFLAG_VISITED_RMY:
+                if (self.header(pointing_to).tid &
+                        GCFLAG_OLD_OUTSIDE_MINIMARKPAGE):
                     pass    # survives, but does not move
                 else:
                     (obj + offset).address[0] = llmemory.NULL
@@ -3035,7 +3066,7 @@
         elif (bool(self.young_rawmalloced_objects) and
               self.young_rawmalloced_objects.contains(obj)):
             # young weakref to a young raw-malloced object
-            if self.header(obj).tid & GCFLAG_VISITED_RMY:
+            if self.header(obj).tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE:
                 surviving = True    # survives, but does not move
             else:
                 surviving = False
diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -1,7 +1,7 @@
 import sys
 from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi
 from rpython.rlib.rarithmetic import LONG_BIT, r_uint
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, not_rpython
 from rpython.rlib.debug import ll_assert, fatalerror
 
 WORD = LONG_BIT // 8
@@ -88,6 +88,7 @@
 class ArenaCollection(object):
     _alloc_flavor_ = "raw"
 
+    @not_rpython
     def __init__(self, arena_size, page_size, small_request_threshold):
         # 'small_request_threshold' is the largest size that we
         # can ask with self.malloc().
@@ -295,6 +296,8 @@
         arena_base = llarena.arena_malloc(self.arena_size, False)
         if not arena_base:
             out_of_memory("out of memory: couldn't allocate the next arena")
+        if not we_are_translated():
+            arena_base.arena._from_minimarkpage = True
         arena_end = arena_base + self.arena_size
         #
         # 'firstpage' points to the first unused page
@@ -570,6 +573,17 @@
         return nblocks - num_initialized_blocks
 
 
+    @not_rpython
+    def _is_inside_minimarkpage(self, hdr):
+        # check that getfakearenaaddress() works and the
+        # arena is flagged as "from minimarkpage"
+        try:
+            arena = llarena.getfakearenaaddress(hdr).arena
+        except RuntimeError:
+            return False
+        return getattr(arena, '_from_minimarkpage', False)
+
+
 # ____________________________________________________________
 # Helpers to go from a pointer to the start of its page
 
diff --git a/rpython/memory/gc/minimarktest.py b/rpython/memory/gc/minimarktest.py
--- a/rpython/memory/gc/minimarktest.py
+++ b/rpython/memory/gc/minimarktest.py
@@ -28,6 +28,7 @@
         ll_assert((nsize & (WORD-1)) == 0, "malloc: size is not aligned")
         #
         result = llarena.arena_malloc(nsize, False)
+        result.arena._from_minimarktest = True
         llarena.arena_reserve(result, size)
         self.all_objects.append((result, nsize))
         self.total_memory_used += nsize
@@ -56,3 +57,10 @@
         self.mass_free_prepare()
         res = self.mass_free_incremental(ok_to_free_func, sys.maxint)
         assert res
+
+    def _is_inside_minimarkpage(self, hdr):
+        try:
+            arena = llarena.getfakearenaaddress(hdr).arena
+        except RuntimeError:
+            return False
+        return getattr(arena, '_from_minimarktest', False)

From pypy.commits at gmail.com  Thu Feb 23 09:31:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 06:31:56 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: fix the non-date
Message-ID: <58aef25c.84522e0a.10dd5.5dc0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5789:8d769027d516
Date: 2017-02-23 15:31 +0100
http://bitbucket.org/pypy/extradoc/changeset/8d769027d516/

Log:	fix the non-date

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -16,7 +16,7 @@
 Robert Zaremba       26?.02/01.03   Ermina
 Ronan Lamy           25.02/04.03    Ermina
 Antonio Cuni         27.02/03.03    Ermina
-Maciej Fijalkowski   26.02/29.02    Ermina
+Maciej Fijalkowski   26.02/01.03    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Thu Feb 23 09:35:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 06:35:21 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: in-progress: try to identify
 all the GCFLAG_VISITED checks on potentially
Message-ID: <58aef329.18532e0a.7645f.15cb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90324:fe4764a96a25
Date: 2017-02-23 15:34 +0100
http://bitbucket.org/pypy/pypy/changeset/fe4764a96a25/

Log:	in-progress: try to identify all the GCFLAG_VISITED checks on
	potentially minimarkpaged-objects

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -113,6 +113,9 @@
 GCFLAG_NO_HEAP_PTRS = first_gcflag << 1
 
 # The following flag is set on surviving objects during a major collection.
+# However, if 'offline_visited_flags' is True, then it is never set on
+# objects in the minimarkpage arenas; instead, we use an offline set of
+# flags.  See get_visited().
 GCFLAG_VISITED      = first_gcflag << 2
 
 # The following flag is set on nursery objects of which we asked the id
@@ -285,6 +288,7 @@
                  card_page_indices=0,
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
+                 offline_visited_flags=False,
                  **kwds):
         "NOT_RPYTHON"
         MovingGCBase.__init__(self, config, **kwds)
@@ -311,6 +315,7 @@
         # 'large_object' limit how big objects can be in the nursery, so
         # it gives a lower bound on the allowed size of the nursery.
         self.nonlarge_max = large_object - 1
+        self.offline_visited_flags = offline_visited_flags
         #
         self.nursery      = llmemory.NULL
         self.nursery_free = llmemory.NULL
@@ -1211,6 +1216,64 @@
             ((r_uint(length) + r_uint((8 << self.card_page_shift) - 1)) >>
              (self.card_page_shift + 3)))
 
+    def get_visited(self, obj):
+        # Return the value of the object's GCFLAG_VISITED flag, noting
+        # that it may be stored offline.  This assumes that 'obj' is an
+        # old object.
+        tid = self.header(obj).tid
+        if self.offline_visited_flags:
+            if tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE == 0:
+                ll_assert((tid & GCFLAG_VISITED) == 0,
+                          "minimarkpage object with in-line GCFLAG_VISITED set")
+                return self.ac.get_visited(obj)
+        return tid & GCFLAG_VISITED != 0
+
+    @specialize.arg(2)
+    def get_visited_or_other_flag(self, obj, other_flag):
+        # Returns 'get_visited(obj) or (tid & other_flag != 0)'
+        # (optimized version)
+        tid = self.header(obj).tid
+        if self.offline_visited_flags:
+            ll_assert(tid & (GCFLAG_OLD_OUTSIDE_MINIMARKPAGE | GCFLAG_VISITED)
+                      != GCFLAG_VISITED, "minimarkpage object with in-line "
+                                         "GCFLAG_VISITED set /2")
+            if (tid & (GCFLAG_OLD_OUTSIDE_MINIMARKPAGE | other_flag)) == 0:
+                return self.ac.get_visited(obj)
+        return (tid & (GCFLAG_VISITED | other_flag)) != 0
+
+    def set_visited(self, obj):
+        # Set the object's GCFLAG_VISITED flag, noting that it may be
+        # stored offline.  This assumes that 'obj' is an old object.
+        tid = self.header(obj).tid
+        if self.offline_visited_flags:
+            if tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE == 0:
+                self.ac.set_visited(obj)
+                return
+        self.header(obj).tid = tid | GCFLAG_VISITED
+
+    @specialize.arg(2)
+    def set_visited_and_other_flag(self, obj, other_flag):
+        # Optimized 'set_visited(obj)' followed by setting 'other_flag' too
+        tid = self.header(obj).tid | other_flag
+        must_set_gcflag_visited = True
+        if self.offline_visited_flags:
+            if tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE == 0:
+                self.ac.set_visited(obj)
+                must_set_gcflag_visited = False
+        if must_set_gcflag_visited:
+            tid |= GCFLAG_VISITED
+        self.header(obj).tid = tid
+
+    def clear_visited(self, obj):
+        # Clear the object's GCFLAG_VISITED flag, noting that it may be
+        # stored offline.  This assumes that 'obj' is an old object.
+        tid = self.header(obj).tid
+        if self.offline_visited_flags:
+            if tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE == 0:
+                self.ac.clear_visited(obj)
+                return
+        self.header(obj).tid = tid & ~GCFLAG_VISITED
+
     def debug_check_consistency(self):
         if self.DEBUG:
             ll_assert(not self.young_rawmalloced_objects,
@@ -1275,7 +1338,7 @@
             ll_assert(False, "unknown gc_state value")
 
     def _debug_check_object_marking(self, obj):
-        if self.header(obj).tid & GCFLAG_VISITED != 0:
+        if self.get_visited(obj):
             # A black object.  Should NEVER point to a white object.
             self.trace(obj, self._debug_check_not_white, None)
             # During marking, all visited (black) objects should always have
@@ -1290,7 +1353,7 @@
 
     def _debug_check_not_white(self, root, ignored):
         obj = root.address[0]
-        if self.header(obj).tid & GCFLAG_VISITED != 0:
+        if self.get_visited(obj):
             pass    # black -> black
         elif (self._debug_objects_to_trace_dict1.contains(obj) or
               self._debug_objects_to_trace_dict2.contains(obj)):
@@ -1355,6 +1418,7 @@
         # the GCFLAG_VISITED should not be set between collections
         ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
                   "unexpected GCFLAG_VISITED")
+        ll_assert(not self.get_visited(obj), "unexpected off-line visited")
 
         # All other invariants from the sweeping phase should still be
         # satisfied.
@@ -1616,7 +1680,7 @@
             # visit shadow to keep it alive
             # XXX seems like it is save to set GCFLAG_VISITED, however
             # should be double checked
-            self.header(shadow).tid |= GCFLAG_VISITED
+            self.set_visited(shadow)
             new_shadow_object_dict.setitem(obj, shadow)
 
     def register_finalizer(self, fq_index, gcobj):
@@ -1787,6 +1851,8 @@
             #
             # clean up object's flags
             obj = cur + size_gc_header
+            ll_assert((self.header(obj).tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE)
+                      != 0, "pinned obj: !old_outside_minimarkpage")
             self.header(obj).tid &= ~GCFLAG_VISITED
             #
             # create a new nursery barrier for the pinned object
@@ -1927,7 +1993,7 @@
                 ll_assert(not self.is_in_nursery(obj),
                           "expected nursery obj in collect_cardrefs_to_nursery")
                 if self.gc_state == STATE_MARKING:
-                    self.header(obj).tid &= ~GCFLAG_VISITED
+                    self.clear_visited(obj)
                     self.more_objects_to_trace.append(obj)
 
 
@@ -1986,7 +2052,7 @@
         # Additionally, ignore pinned objects.
         #
         obj = root.address[0]
-        if (self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_PINNED)) == 0:
+        if not self.get_visited_or_other_flag(obj, GCFLAG_PINNED):
             self.more_objects_to_trace.append(obj)
 
     def _trace_drag_out(self, root, parent):
@@ -2197,11 +2263,11 @@
 
     def _add_to_more_objects_to_trace(self, obj, ignored):
         ll_assert(not self.is_in_nursery(obj), "unexpected nursery obj here")
-        self.header(obj).tid &= ~GCFLAG_VISITED
+        self.clear_visited(obj)
         self.more_objects_to_trace.append(obj)
 
     def _add_to_more_objects_to_trace_if_black(self, obj, ignored):
-        if self.header(obj).tid & GCFLAG_VISITED:
+        if self.get_visited(obj):
             self._add_to_more_objects_to_trace(obj, ignored)
 
     def minor_and_major_collection(self):
@@ -2444,7 +2510,7 @@
         debug_stop("gc-collect-step")
 
     def _sweep_old_objects_pointing_to_pinned(self, obj, new_list):
-        if self.header(obj).tid & GCFLAG_VISITED:
+        if self.get_visited(obj):
             new_list.append(obj)
 
     def _free_if_unvisited(self, hdr):
@@ -2592,12 +2658,12 @@
                   "pinned object in 'objects_to_trace'")
         ll_assert(not self.is_in_nursery(obj),
                   "nursery object in 'objects_to_trace'")
-        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
+        if self.get_visited_or_other_flag(obj, GCFLAG_NO_HEAP_PTRS):
             return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is
         # to also set TRACK_YOUNG_PTRS here, for the write barrier.
-        hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
+        self.set_visited_and_other_flag(obj, GCFLAG_TRACK_YOUNG_PTRS)
 
         if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
             #
@@ -2705,7 +2771,7 @@
         new_objects = self.AddressStack()
         while self.old_objects_with_destructors.non_empty():
             obj = self.old_objects_with_destructors.pop()
-            if self.header(obj).tid & GCFLAG_VISITED:
+            if self.get_visited(obj):
                 # surviving
                 new_objects.append(obj)
             else:
@@ -2744,7 +2810,7 @@
                       "bad finalization state 1")
             if self.header(x).tid & GCFLAG_IGNORE_FINALIZER:
                 continue
-            if self.header(x).tid & GCFLAG_VISITED:
+            if self.get_visited(x):
                 new_with_finalizer.append(x)
                 new_with_finalizer.append(fq_nr)
                 continue
@@ -2796,7 +2862,7 @@
 
     def _finalization_state(self, obj):
         tid = self.header(obj).tid
-        if tid & GCFLAG_VISITED:
+        if self.get_visited(obj):
             if tid & GCFLAG_FINALIZATION_ORDERING:
                 return 2
             else:
@@ -2904,7 +2970,7 @@
         new_with_weakref = self.AddressStack()
         while self.old_objects_with_weakrefs.non_empty():
             obj = self.old_objects_with_weakrefs.pop()
-            if self.header(obj).tid & GCFLAG_VISITED == 0:
+            if not self.get_visited(obj):
                 continue # weakref itself dies
             offset = self.weakpointer_offset(self.get_type_id(obj))
             pointing_to = (obj + offset).address[0]
@@ -2912,8 +2978,8 @@
                       == 0, "registered old weakref should not "
                             "point to a NO_HEAP_PTRS obj")
             tid = self.header(pointing_to).tid
-            if ((tid & (GCFLAG_VISITED | GCFLAG_FINALIZATION_ORDERING)) ==
-                        GCFLAG_VISITED):
+            if (self.get_visited(pointing_to) and
+                    tid & GCFLAG_FINALIZATION_ORDERING == 0):
                 new_with_weakref.append(obj)
             else:
                 (obj + offset).address[0] = llmemory.NULL
@@ -3160,7 +3226,7 @@
         #  * GCFLAG_NO_HEAP_PTRS: immortal object never traced (so far)
         intobj = self._pyobj(pyobject).ob_pypy_link
         obj = llmemory.cast_int_to_adr(intobj)
-        if self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
+        if self.get_visited_or_other_flag(obj, GCFLAG_NO_HEAP_PTRS):
             surviving_list.append(pyobject)
             if surviving_dict:
                 surviving_dict.insertclean(obj, pyobject)

From pypy.commits at gmail.com  Thu Feb 23 10:24:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 23 Feb 2017 07:24:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58aefec5.53532e0a.7a9e3.131b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90325:8a9b287b5544
Date: 2017-02-23 15:24 +0000
http://bitbucket.org/pypy/pypy/changeset/8a9b287b5544/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -160,6 +160,8 @@
 not on PyPy 3.x.  The latter is used to get an app-level unicode string
 by decoding the RPython string, assumed to be utf-8.
 
+.. branch: space-wrap
+
 .. branch: fix_bool_restype
 
 Fix for ``ctypes.c_bool``-returning ctypes functions
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -228,7 +228,10 @@
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
     check_num_args(space, w_args, 0)
-    return space.newint(generic_cpy_call(space, func_len, w_self))
+    res = generic_cpy_call(space, func_len, w_self)
+    if widen(res) == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.newint(res)
 
 def wrap_sq_item(space, w_self, w_args, func):
     func_target = rffi.cast(ssizeargfunc, func)
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -176,7 +176,6 @@
             ("bounce", "METH_NOARGS",
             """
             PyGILState_STATE gilstate;
-            PyThreadState *tstate;
             PyObject *dict;
 
             if (PyEval_ThreadsInitialized() == 0)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -5,7 +5,7 @@
 from rpython.rlib.unicodedata import unicodedb
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import jit
+from rpython.rlib import jit, nonconst
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -373,7 +373,12 @@
                             pos += 1
                             _encodeUCS4(result, ch3)
                             continue
-                    if not allow_surrogates:
+                    # note: if the program only ever calls this with
+                    # allow_surrogates=True, then we'll never annotate
+                    # the following block of code, and errorhandler()
+                    # will never be called.  This causes RPython
+                    # problems.  Avoid it with the nonconst hack.
+                    if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
                                                    s, pos-1, pos)

From pypy.commits at gmail.com  Thu Feb 23 11:44:18 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 23 Feb 2017 08:44:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add extra_test for issue 2289
Message-ID: <58af1162.12142e0a.cb56f.1bff@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90326:88ef793308eb
Date: 2017-02-23 16:43 +0000
http://bitbucket.org/pypy/pypy/changeset/88ef793308eb/

Log:	Add extra_test for issue 2289

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -75,7 +75,7 @@
 ^compiled
 ^.git/
 .git/
-^.hypothesis/
+.hypothesis/
 ^release/
 ^rpython/_cache$
 
diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_unicode.py
@@ -0,0 +1,34 @@
+import pytest
+from hypothesis import strategies as st
+from hypothesis import given, settings, example
+
+from unicodedata import normalize
+
+# For every (n1, n2, n3) triple, applying n1 then n2 must be the same
+# as applying n3.
+# Reference: http://unicode.org/reports/tr15/#Design_Goals
+compositions = [
+    ('NFC', 'NFC', 'NFC'),
+    ('NFC', 'NFD', 'NFD'),
+    ('NFC', 'NFKC', 'NFKC'),
+    ('NFC', 'NFKD', 'NFKD'),
+    ('NFD', 'NFC', 'NFC'),
+    ('NFD', 'NFD', 'NFD'),
+    ('NFD', 'NFKC', 'NFKC'),
+    ('NFD', 'NFKD', 'NFKD'),
+    ('NFKC', 'NFC', 'NFKC'),
+    ('NFKC', 'NFD', 'NFKD'),
+    ('NFKC', 'NFKC', 'NFKC'),
+    ('NFKC', 'NFKD', 'NFKD'),
+    ('NFKD', 'NFC', 'NFKC'),
+    ('NFKD', 'NFD', 'NFKD'),
+    ('NFKD', 'NFKC', 'NFKC'),
+    ('NFKD', 'NFKD', 'NFKD'),
+]
+
+ at pytest.mark.parametrize('norm1, norm2, norm3', compositions)
+ at settings(max_examples=1000)
+ at example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at given(s=st.text())
+def test_composition(s, norm1, norm2, norm3):
+    assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s)

From pypy.commits at gmail.com  Thu Feb 23 11:53:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 23 Feb 2017 08:53:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Add extra_test for issue 2289 (backport
 of 88ef793308eb)
Message-ID: <58af136d.421a190a.cb8af.6c86@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90327:0fbfa6e9cd07
Date: 2017-02-23 16:43 +0000
http://bitbucket.org/pypy/pypy/changeset/0fbfa6e9cd07/

Log:	Add extra_test for issue 2289 (backport of 88ef793308eb)

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -74,7 +74,7 @@
 ^rpython/doc/_build/.*$
 ^compiled
 ^.git/
-^.hypothesis/
+.hypothesis/
 ^release/
 ^rpython/_cache$
 
diff --git a/extra_tests/README.txt b/extra_tests/README.txt
new file mode 100644
--- /dev/null
+++ b/extra_tests/README.txt
@@ -0,0 +1,5 @@
+The tests in this directory are a complement to lib-python/3/test/.
+
+They are meant to run on top of a compiled pypy3 or CPython3.5 in an
+environment containing at least pytest and hypothesis, using a command like
+'pytest extra_tests/'.
diff --git a/extra_tests/pytest.ini b/extra_tests/pytest.ini
new file mode 100644
diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_unicode.py
@@ -0,0 +1,34 @@
+import pytest
+from hypothesis import strategies as st
+from hypothesis import given, settings, example
+
+from unicodedata import normalize
+
+# For every (n1, n2, n3) triple, applying n1 then n2 must be the same
+# as applying n3.
+# Reference: http://unicode.org/reports/tr15/#Design_Goals
+compositions = [
+    ('NFC', 'NFC', 'NFC'),
+    ('NFC', 'NFD', 'NFD'),
+    ('NFC', 'NFKC', 'NFKC'),
+    ('NFC', 'NFKD', 'NFKD'),
+    ('NFD', 'NFC', 'NFC'),
+    ('NFD', 'NFD', 'NFD'),
+    ('NFD', 'NFKC', 'NFKC'),
+    ('NFD', 'NFKD', 'NFKD'),
+    ('NFKC', 'NFC', 'NFKC'),
+    ('NFKC', 'NFD', 'NFKD'),
+    ('NFKC', 'NFKC', 'NFKC'),
+    ('NFKC', 'NFKD', 'NFKD'),
+    ('NFKD', 'NFC', 'NFKC'),
+    ('NFKD', 'NFD', 'NFKD'),
+    ('NFKD', 'NFKC', 'NFKC'),
+    ('NFKD', 'NFKD', 'NFKD'),
+]
+
+ at pytest.mark.parametrize('norm1, norm2, norm3', compositions)
+ at settings(max_examples=1000)
+ at example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at given(s=st.text())
+def test_composition(s, norm1, norm2, norm3):
+    assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s)

From pypy.commits at gmail.com  Thu Feb 23 12:49:15 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 23 Feb 2017 09:49:15 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: implement next_codepoint_pos
Message-ID: <58af209b.11162e0a.87141.15b4@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90328:9bba28798a49
Date: 2017-02-23 18:48 +0100
http://bitbucket.org/pypy/pypy/changeset/9bba28798a49/

Log:	implement next_codepoint_pos

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1032,6 +1032,7 @@
 
         unilist = space.listview_unicode(w_iterable)
         if unilist is not None:
+            xxx
             w_list.strategy = strategy = space.fromcache(UnicodeListStrategy)
             # need to copy because intlist can share with w_iterable
             w_list.lstorage = strategy.erase(unilist[:])
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -299,6 +299,7 @@
     newlist_text = newlist_bytes
 
     def newlist_unicode(self, list_u):
+        xxx
         return self.newlist(list_u)
         return W_ListObject.newlist_unicode(self, list_u)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -471,6 +471,29 @@
             sb.append(unwrapped[i])
         return self._new(sb.build(), lgt)
 
+    @unwrap_spec(keepends=bool)
+    def descr_splitlines(self, space, keepends=False):
+        value = self._val(space)
+        length = len(value)
+        strs = []
+        pos = 0
+        while pos < length:
+            sol = pos
+            while pos < length and not self._islinebreak(value[pos]):
+                pos += 1
+            eol = pos
+            pos += 1
+            # read CRLF as one line break
+            if pos < length and value[eol] == '\r' and value[pos] == '\n':
+                pos += 1
+            if keepends:
+                eol = pos
+            strs.append(value[sol:eol])
+        if pos < length:
+            strs.append(value[pos:length])
+        return self._newlist_unwrapped(space, strs)
+
+
 def wrapunicode(space, uni):
     return W_UnicodeObject(uni)
 
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -29,6 +29,15 @@
                 chr((0x80 | (code & 0x3f)))), lgt
     raise ValueError
 
+def next_codepoint_pos(code, pos):
+    """ Gives the position of the next codepoint after pos, -1
+    if it's the last one (assumes valid utf8)
+    """
+    chr1 = ord(code[pos])
+    if chr1 < 0x80:
+        return pos + 1
+    return pos + ord(runicode._utf8_code_length[chr1 - 0x80])
+
 class AsciiCheckError(Exception):
     def __init__(self, pos):
         self.pos = pos
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -1,5 +1,5 @@
 
-from hypothesis import given, strategies, settings
+from hypothesis import given, strategies, settings, example
 
 from rpython.rlib import rutf8, runicode
 
@@ -30,7 +30,7 @@
 @given(strategies.binary())
 def test_str_check_utf8(s):
     try:
-        u = s.decode("utf8")
+        u, _ = runicode.str_decode_utf_8(s, len(s), None, final=True)
         valid = True
     except UnicodeDecodeError as e:
         valid = False
@@ -49,4 +49,18 @@
 @given(strategies.binary())
 def test_str_decode_raw_utf8_escape(uni):
     return # XXX fix details
-    rutf8.str_decode_raw_utf8_escape(uni, len(uni), None)
\ No newline at end of file
+    rutf8.str_decode_raw_utf8_escape(uni, len(uni), None)
+
+ at given(strategies.characters())
+def test_next_pos(uni):
+    skips = []
+    for elem in uni:
+        skips.append(len(elem.encode('utf8')))
+    pos = 0
+    i = 0
+    utf8 = uni.encode('utf8')
+    while pos < len(utf8):
+        new_pos = rutf8.next_codepoint_pos(utf8, pos)
+        assert new_pos - pos == skips[i]
+        i += 1
+        pos = new_pos

From pypy.commits at gmail.com  Thu Feb 23 12:53:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 09:53:21 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: in-progress,
 seems to translate and run simple examples
Message-ID: <58af2191.51d2190a.bc751.704b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90329:e63f85f1dc45
Date: 2017-02-23 18:52 +0100
http://bitbucket.org/pypy/pypy/changeset/e63f85f1dc45/

Log:	in-progress, seems to translate and run simple examples

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -92,6 +92,9 @@
                  }),
     BoolOption("gcremovetypeptr", "Remove the typeptr from every object",
                default=IS_64_BITS, cmdline="--gcremovetypeptr"),
+    BoolOption("gcforkfriendly", "A fork-friendly variant of incminimark",
+               default=False, cmdline="--gcforkfriendly",
+               requires=[("translation.gc", "incminimark")]),
     ChoiceOption("gcrootfinder",
                  "Strategy for finding GC Roots (framework GCs only)",
                  ["n/a", "shadowstack", "asmgcc"],
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -288,7 +288,6 @@
                  card_page_indices=0,
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
-                 offline_visited_flags=False,
                  **kwds):
         "NOT_RPYTHON"
         MovingGCBase.__init__(self, config, **kwds)
@@ -315,7 +314,6 @@
         # 'large_object' limit how big objects can be in the nursery, so
         # it gives a lower bound on the allowed size of the nursery.
         self.nonlarge_max = large_object - 1
-        self.offline_visited_flags = offline_visited_flags
         #
         self.nursery      = llmemory.NULL
         self.nursery_free = llmemory.NULL
@@ -325,11 +323,13 @@
         self.extra_threshold = 0
         #
         # The ArenaCollection() handles the nonmovable objects allocation.
+        self.offline_visited_flags = config.gcforkfriendly
         if ArenaCollectionClass is None:
             from rpython.memory.gc import minimarkpage
             ArenaCollectionClass = minimarkpage.ArenaCollection
         self.ac = ArenaCollectionClass(arena_size, page_size,
-                                       small_request_threshold)
+                                       small_request_threshold,
+                                       self.offline_visited_flags)
         #
         # Used by minor collection: a list of (mostly non-young) objects that
         # (may) contain a pointer to a young object.  Populated by
diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -2,7 +2,9 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi
 from rpython.rlib.rarithmetic import LONG_BIT, r_uint
 from rpython.rlib.objectmodel import we_are_translated, not_rpython
+from rpython.rlib.objectmodel import always_inline
 from rpython.rlib.debug import ll_assert, fatalerror
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 WORD = LONG_BIT // 8
 NULL = llmemory.NULL
@@ -89,7 +91,16 @@
     _alloc_flavor_ = "raw"
 
     @not_rpython
-    def __init__(self, arena_size, page_size, small_request_threshold):
+    def __init__(self, arena_size, page_size, small_request_threshold,
+                 offline_visited_flags=False):
+        # If we ask for 'offline_visited_flags', then we'll allocate
+        # arenas that are always OFFL_RATIO pages in size, and fully aligned.
+        # In this case, page_size should be 4096 or 8192.  The first page is
+        # used for the offline_visited_flags.  See rpy_allocate_new_arena().
+        if offline_visited_flags:
+            arena_size = OFFL_ARENA_SIZE - OFFL_SYSTEM_PAGE_SIZE
+        self.offline_visited_flags = offline_visited_flags
+        #
         # 'small_request_threshold' is the largest size that we
         # can ask with self.malloc().
         self.arena_size = arena_size
@@ -290,21 +301,30 @@
         if not we_are_translated():
             for a in self._all_arenas():
                 assert a.nfreepages == 0
+
+        if not self.offline_visited_flags:
+            #
+            # 'arena_base' points to the start of malloced memory; it might not
+            # be a page-aligned address
+            arena_base = llarena.arena_malloc(self.arena_size, False)
+            if not arena_base:
+                out_of_memory("out of memory: couldn't allocate the next arena")
+            arena_end = arena_base + self.arena_size
+            #
+            # 'firstpage' points to the first unused page
+            firstpage = start_of_page(arena_base + self.page_size - 1,
+                                      self.page_size)
+        else:
+            assert OFFL_SYSTEM_PAGE_SIZE == llarena.posixpagesize.get()
+            arena_base = rpy_allocate_new_arena()
+            arena_end = arena_base + OFFL_ARENA_SIZE
+            firstpage = arena_base + max(self.page_size, OFFL_SYSTEM_PAGE_SIZE)
         #
-        # 'arena_base' points to the start of malloced memory; it might not
-        # be a page-aligned address
-        arena_base = llarena.arena_malloc(self.arena_size, False)
-        if not arena_base:
-            out_of_memory("out of memory: couldn't allocate the next arena")
+        # 'npages' is the number of full pages just allocated
+        npages = (arena_end - firstpage) // self.page_size
+
         if not we_are_translated():
             arena_base.arena._from_minimarkpage = True
-        arena_end = arena_base + self.arena_size
-        #
-        # 'firstpage' points to the first unused page
-        firstpage = start_of_page(arena_base + self.page_size - 1,
-                                  self.page_size)
-        # 'npages' is the number of full pages just allocated
-        npages = (arena_end - firstpage) // self.page_size
         #
         # Allocate an ARENA object and initialize it
         arena = lltype.malloc(ARENA, flavor='raw', track_allocation=False)
@@ -399,7 +419,10 @@
                     #
                     # The whole arena is empty.  Free it.
                     llarena.arena_reset(arena.base, self.arena_size, 4)
-                    llarena.arena_free(arena.base)
+                    if not self.offline_visited_flags:
+                        llarena.arena_free(arena.base)
+                    else:
+                        rpy_free_arena(arena.base)
                     lltype.free(arena, flavor='raw', track_allocation=False)
                     #
                 else:
@@ -510,6 +533,9 @@
         surviving = 0    # initially
         skip_free_blocks = page.nfree
         #
+        if self.offline_visited_flags:
+            ok_to_free_func = self.get_unvisited
+        #
         while True:
             #
             if obj == freeblock:
@@ -555,6 +581,9 @@
         # Update the global total size of objects.
         self.total_memory_used += r_uint(surviving * block_size)
         #
+        if self.offline_visited_flags:
+            self.reset_block_of_visited_flags_for_one_page(page)
+        #
         # Return the number of surviving objects.
         return surviving
 
@@ -584,6 +613,50 @@
         return getattr(arena, '_from_minimarkpage', False)
 
 
+    @staticmethod
+    @always_inline
+    def get_visited(obj):
+        numeric = rffi.cast(lltype.Unsigned, obj)
+        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
+        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
+        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
+        return (ord(base[ofs]) & singlebit) != 0
+
+    @staticmethod
+    @always_inline
+    def set_visited(obj):
+        numeric = rffi.cast(lltype.Unsigned, obj)
+        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
+        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
+        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
+        base[ofs] = chr(ord(base[ofs]) | singlebit)
+
+    @staticmethod
+    @always_inline
+    def clear_visited(obj):
+        numeric = rffi.cast(lltype.Unsigned, obj)
+        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
+        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
+        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
+        base[ofs] = chr(ord(base[ofs]) & ~singlebit)
+
+    @staticmethod
+    @always_inline
+    def get_unvisited(obj):
+        numeric = rffi.cast(lltype.Unsigned, obj)
+        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
+        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
+        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
+        return (ord(base[ofs]) & singlebit) == 0
+
+    @staticmethod
+    def reset_block_of_visited_flags_for_one_page(page):
+        numeric = rffi.cast(lltype.Unsigned, page)
+        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
+        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
+        rpy_memset(base, 0, OFFL_SYSTEM_PAGE_SIZE // OFFL_RATIO)
+
+
 # ____________________________________________________________
 # Helpers to go from a pointer to the start of its page
 
@@ -615,3 +688,95 @@
     exception gracefully.
     """
     fatalerror(errmsg)
+
+
+# ____________________________________________________________
+# Helpers for the 'offline visited flags' mode
+
+
+# xxx make the number 4096 not hard-coded, but it has implications on
+# the other numbers too
+OFFL_SYSTEM_PAGE_SIZE = 4096
+OFFL_RATIO = WORD * 2 * 8
+OFFL_ARENA_SIZE = OFFL_SYSTEM_PAGE_SIZE * OFFL_RATIO
+
+# Idea: mmap N bytes, where N is just smaller than '2 * arena_size'.
+# This is just the right size to ensure that the allocated region
+# contains exactly one block of 'arena_size' pages that is fully
+# aligned to an address multiple of 'arena_size'.  Then we munmap
+# the extra bits at both ends.  This approach should ensure that
+# after the first arena was allocated, the next one is likely to be
+# placed by the system either just before or just after it; when
+# it is the case, the side that connects with the previous arena
+# is aligned already, and so we only have to remove the extra bit
+# at the other end.  This should ensure that our aligned arenas
+# grow next to each other (and in a single VMA, in Linux terms).
+
+# The number OFFL_RATIO comes from the fact that we can use one
+# visited bit for every two words of memory.  Most objects are at
+# least two words in length.  If we have a page that contains
+# single-word objects, then the visited bits clash: the same bit
+# is used for two objects.  However, that's a very minor problem,
+# because the objects are too small to contain further references
+# anyway, so at most we leak one of the two objects when the other
+# is still in use (i.e. at most one word of memory per word-sized
+# object alive).  So, OFFL_RATIO is equal to 64 on 32-bit and 128
+# on 64-bit machines.  That's also equal to the number of pages in
+# one arena: we reserve the first page for visited bits, and all
+# remaining pages are used for real objects.
+
+eci = ExternalCompilationInfo(
+    post_include_bits=[
+        'RPY_EXTERN void *rpy_allocate_new_arena(void);',
+        'RPY_EXTERN void rpy_free_arena(void *);',
+        'RPY_EXTERN char rpy_get_visited(void *);',
+        'RPY_EXTERN void rpy_set_visited(void *);',
+        'RPY_EXTERN void rpy_clear_visited(void *);',
+        ],
+    separate_module_sources=['''
+#include <stdlib.h>
+#include <sys/mman.h>
+
+#define OFFL_SYSTEM_PAGE_SIZE  %(OFFL_SYSTEM_PAGE_SIZE)d
+#define OFFL_RATIO             %(OFFL_RATIO)d
+#define OFFL_ARENA_SIZE        %(OFFL_ARENA_SIZE)d
+
+RPY_EXTERN void *rpy_allocate_new_arena(void)
+{
+    size_t arena_size = OFFL_ARENA_SIZE;
+    size_t map_size = arena_size * 2 - OFFL_SYSTEM_PAGE_SIZE;
+    void *p = mmap(NULL, map_size, PROT_READ | PROT_WRITE,
+                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (p == MAP_FAILED) {
+        perror("Fatal RPython error: mmap");
+        abort();
+    }
+
+    char *result = (char *)((((long)p) + arena_size - 1) & ~(arena_size-1));
+    if (result > (char *)p)
+        munmap(p, result - (char *)p);
+    long free_end = ((char *)p + map_size) - (result + arena_size);
+    if (free_end > 0)
+        munmap(result + arena_size, free_end);
+
+    /* 'result' is freshly mmap()ed so it contains zeroes at this point */
+    return result;
+}
+
+RPY_EXTERN void rpy_free_arena(void *base)
+{
+    munmap(base, OFFL_ARENA_SIZE);
+}
+''' % globals()])
+
+rpy_allocate_new_arena = rffi.llexternal(
+    'rpy_allocate_new_arena', [], llmemory.Address,
+    compilation_info=eci, _nowrapper=True)
+
+rpy_free_arena = rffi.llexternal(
+    'rpy_free_arena', [llmemory.Address], lltype.Void,
+    compilation_info=eci, _nowrapper=True)
+
+rpy_memset = rffi.llexternal(
+    'memset', [rffi.CCHARP, lltype.Signed, lltype.Signed], lltype.Void,
+    _nowrapper=True)
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -25,6 +25,7 @@
     taggedpointers = False
     GC_CAN_MOVE = False
     GC_CAN_SHRINK_ARRAY = False
+    gcforkfriendly = False
 
     _isolated_func = None
     c_allfuncs = None
@@ -44,7 +45,8 @@
 
         t = Translation(main, gc=cls.gcpolicy,
                         taggedpointers=cls.taggedpointers,
-                        gcremovetypeptr=cls.removetypeptr)
+                        gcremovetypeptr=cls.removetypeptr,
+                        gcforkfriendly=cls.gcforkfriendly)
         t.disable(['backendopt'])
         t.set_backend_extra_options(c_debug_defines=True)
         t.rtype()
@@ -1667,6 +1669,7 @@
 
 class TestIncrementalMiniMarkGC(TestMiniMarkGC):
     gcpolicy = "incminimark"
+    gcforkfriendly = True
 
     def define_random_pin(self):
         class A:

From pypy.commits at gmail.com  Thu Feb 23 13:39:27 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 23 Feb 2017 10:39:27 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: implement splitlines,
 strange unreachable code?
Message-ID: <58af2c5f.02a5190a.90486.71b8@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90331:4f3f66d1551a
Date: 2017-02-23 19:38 +0100
http://bitbucket.org/pypy/pypy/changeset/4f3f66d1551a/

Log:	implement splitlines, strange unreachable code?

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -608,6 +608,7 @@
                 eol = pos
             strs.append(value[sol:eol])
         if pos < length:
+            # XXX is this code reachable ever?
             strs.append(value[pos:length])
         return self._newlist_unwrapped(space, strs)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -185,8 +185,8 @@
     def _iscased(self, ch):
         return unicodedb.iscased(ord(ch))
 
-    def _islinebreak(self, ch):
-        return unicodedb.islinebreak(ord(ch))
+    def _islinebreak(self, s, pos):
+        return rutf8.check_newline_utf8(s, pos)
 
     def _upper(self, ch):
         return unichr(unicodedb.toupper(ord(ch)))
@@ -475,23 +475,27 @@
     def descr_splitlines(self, space, keepends=False):
         value = self._val(space)
         length = len(value)
-        strs = []
+        strs_w = []
         pos = 0
         while pos < length:
             sol = pos
+            lgt = 0
             while pos < length and not self._islinebreak(value, pos):
                 pos = rutf8.next_codepoint_pos(value, pos)
+                lgt += 1
             eol = pos
-            pos += 1
+            if pos < length:
+                pos = rutf8.next_codepoint_pos(value, pos)
             # read CRLF as one line break
             if pos < length and value[eol] == '\r' and value[pos] == '\n':
                 pos += 1
+                if keepends:
+                    lgt += 1
             if keepends:
                 eol = pos
-            strs.append(value[sol:eol])
-        if pos < length:
-            strs.append(value[pos:length])
-        return self._newlist_unwrapped(space, strs)
+                lgt += 2
+            strs_w.append(W_UnicodeObject(value[sol:eol], lgt))
+        return space.newlist(strs_w)
 
 
 def wrapunicode(space, uni):

From pypy.commits at gmail.com  Thu Feb 23 13:39:24 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 23 Feb 2017 10:39:24 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: give up and write the regex by
 hand, check in a test
Message-ID: <58af2c5c.87472e0a.4498.1bd1@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90330:62c7e93c717d
Date: 2017-02-23 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/62c7e93c717d/

Log:	give up and write the regex by hand, check in a test

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -479,8 +479,8 @@
         pos = 0
         while pos < length:
             sol = pos
-            while pos < length and not self._islinebreak(value[pos]):
-                pos += 1
+            while pos < length and not self._islinebreak(value, pos):
+                pos = rutf8.next_codepoint_pos(value, pos)
             eol = pos
             pos += 1
             # read CRLF as one line break
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -50,6 +50,23 @@
 def default_unicode_error_check(*args):
     xxx
 
+def check_newline_utf8(s, pos):
+    chr1 = ord(s[pos])
+    if 0xa <= chr1 <= 0xd:
+        return True
+    if 0x1c <= chr1 <= 0x1e:
+        return True
+    if chr1 == 0xc2:
+        chr2 = ord(s[pos + 1])
+        return chr2 == 0x85
+    elif chr1 == 0xe2:
+        chr2 = ord(s[pos + 1])
+        if chr2 != 0x80:
+            return False
+        chr3 = ord(s[pos + 2])
+        return chr3 == 0xa8 or chr3 == 0xa9
+    return False
+
 # if you can't use the @elidable version, call str_check_utf8_impl()
 # directly
 @jit.elidable
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -1,4 +1,5 @@
 
+import sys
 from hypothesis import given, strategies, settings, example
 
 from rpython.rlib import rutf8, runicode
@@ -64,3 +65,10 @@
         assert new_pos - pos == skips[i]
         i += 1
         pos = new_pos
+
+def test_check_newline_utf8():
+    for i in xrange(sys.maxunicode):
+        if runicode.unicodedb.islinebreak(i):
+            assert rutf8.check_newline_utf8(unichr(i).encode('utf8'), 0)
+        else:
+            assert not rutf8.check_newline_utf8(unichr(i).encode('utf8'), 0)

From pypy.commits at gmail.com  Thu Feb 23 14:16:03 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 23 Feb 2017 11:16:03 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: implement zfill
Message-ID: <58af34f3.15502e0a.10271.2815@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90332:429e8e21cd00
Date: 2017-02-23 20:15 +0100
http://bitbucket.org/pypy/pypy/changeset/429e8e21cd00/

Log:	implement zfill

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -151,7 +151,7 @@
 
     def _chr(self, char):
         assert len(char) == 1
-        return unicode(char)[0]
+        return char[0]
 
     _builder = StringBuilder
 
@@ -497,6 +497,26 @@
             strs_w.append(W_UnicodeObject(value[sol:eol], lgt))
         return space.newlist(strs_w)
 
+    @unwrap_spec(width=int)
+    def descr_zfill(self, space, width):
+        selfval = self._val(space)
+        if len(selfval) == 0:
+            return self._new(self._multi_chr(self._chr('0')) * width, width)
+        num_zeros = width - self._length
+        if num_zeros <= 0:
+            # cannot return self, in case it is a subclass of str
+            return self._new(selfval, self._length)
+        builder = self._builder(num_zeros + len(selfval))
+        if len(selfval) > 0 and (selfval[0] == '+' or selfval[0] == '-'):
+            # copy sign to first position
+            builder.append(selfval[0])
+            start = 1
+        else:
+            start = 0
+        builder.append_multiple_char(self._chr('0'), num_zeros)
+        builder.append_slice(selfval, start, len(selfval))
+        return self._new(builder.build(), width)
+
 
 def wrapunicode(space, uni):
     return W_UnicodeObject(uni)

From pypy.commits at gmail.com  Thu Feb 23 14:33:14 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 23 Feb 2017 11:33:14 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: split and rsplit (inefficient)
Message-ID: <58af38fa.43ca190a.c250d.77af@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90333:b17610fd28e3
Date: 2017-02-23 20:32 +0100
http://bitbucket.org/pypy/pypy/changeset/b17610fd28e3/

Log:	split and rsplit (inefficient)

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -608,7 +608,7 @@
                 eol = pos
             strs.append(value[sol:eol])
         if pos < length:
-            # XXX is this code reachable ever?
+            # XXX is this code reachable?
             strs.append(value[pos:length])
         return self._newlist_unwrapped(space, strs)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -4,7 +4,7 @@
     compute_hash, compute_unique_id, import_from_mixin,
     enforceargs, newlist_hint)
 from rpython.rlib.buffer import StringBuffer
-from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
+from rpython.rlib.rstring import StringBuilder, split, rsplit
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
@@ -120,6 +120,8 @@
         return W_UnicodeObject.EMPTY
 
     def _len(self):
+        if self._length == -1:
+            self._length = self._compute_length()
         return self._length
 
     _val = utf8_w
@@ -438,13 +440,13 @@
                 return space.w_NotImplemented
             raise
         return W_UnicodeObject(self._utf8 + w_other._utf8,
-                               self._length + w_other._length)
+                               self._len() + w_other._len())
 
     @jit.look_inside_iff(lambda self, space, list_w, size:
                          jit.loop_unrolling_heuristic(list_w, size))
     def _str_join_many_items(self, space, list_w, size):
         value = self._utf8
-        lgt = self._length * (size - 1)
+        lgt = self._len() * (size - 1)
 
         prealloc_size = len(value) * (size - 1)
         unwrapped = newlist_hint(size)
@@ -502,10 +504,10 @@
         selfval = self._val(space)
         if len(selfval) == 0:
             return self._new(self._multi_chr(self._chr('0')) * width, width)
-        num_zeros = width - self._length
+        num_zeros = width - self._len()
         if num_zeros <= 0:
             # cannot return self, in case it is a subclass of str
-            return self._new(selfval, self._length)
+            return self._new(selfval, self._len())
         builder = self._builder(num_zeros + len(selfval))
         if len(selfval) > 0 and (selfval[0] == '+' or selfval[0] == '-'):
             # copy sign to first position
@@ -517,6 +519,36 @@
         builder.append_slice(selfval, start, len(selfval))
         return self._new(builder.build(), width)
 
+    @unwrap_spec(maxsplit=int)
+    def descr_split(self, space, w_sep=None, maxsplit=-1):
+        # XXX maybe optimize?
+        res = []
+        value = self._val(space)
+        if space.is_none(w_sep):
+            res = split(value, maxsplit=maxsplit)
+            return space.newlist([W_UnicodeObject(s, -1) for s in res])
+
+        by = self._op_val(space, w_sep)
+        if len(by) == 0:
+            raise oefmt(space.w_ValueError, "empty separator")
+        res = split(value, by, maxsplit)
+
+        return space.newlist([W_UnicodeObject(s, -1) for s in res])
+
+    @unwrap_spec(maxsplit=int)
+    def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
+        res = []
+        value = self._val(space)
+        if space.is_none(w_sep):
+            res = rsplit(value, maxsplit=maxsplit)
+            return space.newlist([W_UnicodeObject(s, -1) for s in res])
+
+        by = self._op_val(space, w_sep)
+        if len(by) == 0:
+            raise oefmt(space.w_ValueError, "empty separator")
+        res = rsplit(value, by, maxsplit)
+
+        return space.newlist([W_UnicodeObject(s, -1) for s in res])
 
 def wrapunicode(space, uni):
     return W_UnicodeObject(uni)

From pypy.commits at gmail.com  Fri Feb 24 02:22:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 23:22:54 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: Speed tweaks including a bug
 fix
Message-ID: <58afdf4e.12142e0a.cb56f.3a90@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90334:6a642098cb7f
Date: 2017-02-24 08:22 +0100
http://bitbucket.org/pypy/pypy/changeset/6a642098cb7f/

Log:	Speed tweaks including a bug fix

diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -533,8 +533,8 @@
         surviving = 0    # initially
         skip_free_blocks = page.nfree
         #
-        if self.offline_visited_flags:
-            ok_to_free_func = self.get_unvisited
+        vblocknext = r_uint(0)
+        vblock = rffi.cast(rffi.ULONGLONG, -1)
         #
         while True:
             #
@@ -558,7 +558,16 @@
                 ll_assert(freeblock > obj,
                           "freeblocks are linked out of order")
                 #
-                if ok_to_free_func(obj):
+                if self.offline_visited_flags:
+                    if self.visited_flags_limit_reached(obj, vblocknext):
+                        vblocknext = (
+                            self.get_64bit_limit_for_next_visited_flags(obj))
+                        vblock = self.fetch_64bit_visited_flags(obj)
+                    vblockmask = self.get_64bit_mask_visited_flag(obj)
+                    ok = not (vblock & vblockmask)
+                else:
+                    ok = ok_to_free_func(obj)
+                if ok:
                     #
                     # The object should die.
                     llarena.arena_reset(obj, _dummy_size(block_size), 0)
@@ -581,9 +590,6 @@
         # Update the global total size of objects.
         self.total_memory_used += r_uint(surviving * block_size)
         #
-        if self.offline_visited_flags:
-            self.reset_block_of_visited_flags_for_one_page(page)
-        #
         # Return the number of surviving objects.
         return surviving
 
@@ -615,46 +621,71 @@
 
     @staticmethod
     @always_inline
-    def get_visited(obj):
+    def get_64bit_ptr_visited_flags(obj):
         numeric = rffi.cast(lltype.Unsigned, obj)
         base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
-        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
-        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
-        return (ord(base[ofs]) & singlebit) != 0
+        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 8)
+        return rffi.cast(rffi.ULONGLONGP, rffi.ptradd(base, ofs))
+
+    @staticmethod
+    @always_inline
+    def get_64bit_mask_visited_flag(obj):
+        numeric = rffi.cast(lltype.Unsigned, obj)
+        shift = (numeric // (OFFL_RATIO/8)) & 63
+        return rffi.cast(rffi.ULONGLONG, 1) << shift
+
+    @staticmethod
+    @always_inline
+    def get_visited(obj):
+        """test the visited flag corresponding to 'obj'"""
+        p = ArenaCollection.get_64bit_ptr_visited_flags(obj)
+        mask = ArenaCollection.get_64bit_mask_visited_flag(obj)
+        return (p[0] & mask) != 0
 
     @staticmethod
     @always_inline
     def set_visited(obj):
-        numeric = rffi.cast(lltype.Unsigned, obj)
-        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
-        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
-        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
-        base[ofs] = chr(ord(base[ofs]) | singlebit)
+        """set (to 1) the visited flag corresponding to 'obj'"""
+        p = ArenaCollection.get_64bit_ptr_visited_flags(obj)
+        mask = ArenaCollection.get_64bit_mask_visited_flag(obj)
+        p[0] |= mask
 
     @staticmethod
     @always_inline
     def clear_visited(obj):
-        numeric = rffi.cast(lltype.Unsigned, obj)
-        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
-        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
-        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
-        base[ofs] = chr(ord(base[ofs]) & ~singlebit)
+        """clear the visited flag corresponding to 'obj'"""
+        # (Note: should not be used too often.  Due to the fact that the
+        # same flag might be used for two objects if they are a single
+        # word each, this might occasionally clear too much.  It is
+        # still fine in this case because we clear the flag only to
+        # force re-visiting the object later during major collection)
+        p = ArenaCollection.get_64bit_ptr_visited_flags(obj)
+        mask = ArenaCollection.get_64bit_mask_visited_flag(obj)
+        p[0] &= ~mask
 
     @staticmethod
     @always_inline
-    def get_unvisited(obj):
+    def get_64bit_limit_for_next_visited_flags(obj):
+        """get a result that encodes the last possible position of an
+        object after 'obj' where the corresponding visited flag is in
+        the same 64-bit block as 'obj'."""
         numeric = rffi.cast(lltype.Unsigned, obj)
-        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
-        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
-        singlebit = 1 << ((numeric // (OFFL_RATIO/8)) & 7)
-        return (ord(base[ofs]) & singlebit) == 0
+        return numeric | (64 * OFFL_RATIO / 8 - 1)
 
     @staticmethod
-    def reset_block_of_visited_flags_for_one_page(page):
-        numeric = rffi.cast(lltype.Unsigned, page)
-        base = rffi.cast(rffi.CCHARP, numeric & ~(OFFL_ARENA_SIZE - 1))
-        ofs = (numeric // OFFL_RATIO) & (OFFL_SYSTEM_PAGE_SIZE - 1)
-        rpy_memset(base, 0, OFFL_SYSTEM_PAGE_SIZE // OFFL_RATIO)
+    @always_inline
+    def visited_flags_limit_reached(obj, vblocknext):
+        """return True if 'obj' is beyond the limit computed by
+        get_64bit_limit_for_next_visited_flags()."""
+        return rffi.cast(lltype.Unsigned, obj) > vblocknext
+
+    @staticmethod
+    @always_inline
+    def fetch_64bit_visited_flags(obj):
+        p = ArenaCollection.get_64bit_ptr_visited_flags(obj)
+        result = p[0]
+        p[0] = rffi.cast(rffi.ULONGLONG, 0)
+        return result
 
 
 # ____________________________________________________________
@@ -748,7 +779,7 @@
     void *p = mmap(NULL, map_size, PROT_READ | PROT_WRITE,
                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     if (p == MAP_FAILED) {
-        perror("Fatal RPython error: mmap");
+        perror("Fatal RPython error: out of memory");
         abort();
     }
 
@@ -776,7 +807,3 @@
 rpy_free_arena = rffi.llexternal(
     'rpy_free_arena', [llmemory.Address], lltype.Void,
     compilation_info=eci, _nowrapper=True)
-
-rpy_memset = rffi.llexternal(
-    'memset', [rffi.CCHARP, lltype.Signed, lltype.Signed], lltype.Void,
-    _nowrapper=True)

From pypy.commits at gmail.com  Fri Feb 24 02:41:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 23:41:55 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test on my laptop
Message-ID: <58afe3c3.4fd0190a.58a88.8aa1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90335:c5721c39a21f
Date: 2017-02-24 08:39 +0100
http://bitbucket.org/pypy/pypy/changeset/c5721c39a21f/

Log:	fix test on my laptop

diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1730,7 +1730,11 @@
                      (ulimitv, ' '.join(args),)]
             popen = subprocess.Popen(args1, stderr=subprocess.PIPE)
             _, child_stderr = popen.communicate()
-            assert popen.wait() == 134     # aborted
+            assert popen.wait() in (-6, 134)     # aborted
+            # note: it seems that on some systems we get 134 and on
+            # others we get -6.  Bash is supposed to translate the
+            # SIGABRT (signal 6) from the subprocess into the exit 
+            # code 128+6, but I guess it may not always do so.
             assert 'out of memory:' in child_stderr
             return '42'
         #

From pypy.commits at gmail.com  Fri Feb 24 02:41:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 23:41:59 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: hg merge default
Message-ID: <58afe3c7.15502e0a.10271.3f86@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90336:7baaab0977a5
Date: 2017-02-24 08:40 +0100
http://bitbucket.org/pypy/pypy/changeset/7baaab0977a5/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -151,6 +151,8 @@
 not on PyPy 3.x.  The latter is used to get an app-level unicode string
 by decoding the RPython string, assumed to be utf-8.
 
+.. branch: space-wrap
+
 .. branch: fix_bool_restype
 
 Fix for ``ctypes.c_bool``-returning ctypes functions
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -420,7 +420,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         _args = arguments.from_object(space, w_args)
@@ -497,7 +497,7 @@
         w_decorator_list = get_field(space, w_node, 'decorator_list', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
         bases_w = space.unpackiterable(w_bases)
@@ -1318,7 +1318,7 @@
         w_level = get_field(space, w_node, 'level', True)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _module = space.str_or_None_w(w_module)
+        _module = space.realtext_w(w_module) if not space.is_none(w_module) else None
         names_w = space.unpackiterable(w_names)
         _names = [alias.from_object(space, w_item) for w_item in names_w]
         _level = space.int_w(w_level)
@@ -1413,7 +1413,7 @@
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
         names_w = space.unpackiterable(w_names)
-        _names = [space.realstr_w(w_item) for w_item in names_w]
+        _names = [space.realtext_w(w_item) for w_item in names_w]
         _lineno = space.int_w(w_lineno)
         _col_offset = space.int_w(w_col_offset)
         return Global(_names, _lineno, _col_offset)
@@ -2495,7 +2495,7 @@
         _value = expr.from_object(space, w_value)
         if _value is None:
             raise_required_value(space, w_node, 'value')
-        _attr = space.realstr_w(w_attr)
+        _attr = space.realtext_w(w_attr)
         if _attr is None:
             raise_required_value(space, w_node, 'attr')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -2592,7 +2592,7 @@
         w_ctx = get_field(space, w_node, 'ctx', False)
         w_lineno = get_field(space, w_node, 'lineno', False)
         w_col_offset = get_field(space, w_node, 'col_offset', False)
-        _id = space.realstr_w(w_id)
+        _id = space.realtext_w(w_id)
         if _id is None:
             raise_required_value(space, w_node, 'id')
         _ctx = expr_context.from_object(space, w_ctx)
@@ -3415,8 +3415,8 @@
         w_defaults = get_field(space, w_node, 'defaults', False)
         args_w = space.unpackiterable(w_args)
         _args = [expr.from_object(space, w_item) for w_item in args_w]
-        _vararg = space.str_or_None_w(w_vararg)
-        _kwarg = space.str_or_None_w(w_kwarg)
+        _vararg = space.realtext_w(w_vararg) if not space.is_none(w_vararg) else None
+        _kwarg = space.realtext_w(w_kwarg) if not space.is_none(w_kwarg) else None
         defaults_w = space.unpackiterable(w_defaults)
         _defaults = [expr.from_object(space, w_item) for w_item in defaults_w]
         return arguments(_args, _vararg, _kwarg, _defaults)
@@ -3448,7 +3448,7 @@
     def from_object(space, w_node):
         w_arg = get_field(space, w_node, 'arg', False)
         w_value = get_field(space, w_node, 'value', False)
-        _arg = space.realstr_w(w_arg)
+        _arg = space.realtext_w(w_arg)
         if _arg is None:
             raise_required_value(space, w_node, 'arg')
         _value = expr.from_object(space, w_value)
@@ -3482,10 +3482,10 @@
     def from_object(space, w_node):
         w_name = get_field(space, w_node, 'name', False)
         w_asname = get_field(space, w_node, 'asname', True)
-        _name = space.realstr_w(w_name)
+        _name = space.realtext_w(w_name)
         if _name is None:
             raise_required_value(space, w_node, 'name')
-        _asname = space.str_or_None_w(w_asname)
+        _asname = space.realtext_w(w_asname) if not space.is_none(w_asname) else None
         return alias(_name, _asname)
 
 State.ast_type('alias', 'AST', ['name', 'asname'])
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -150,8 +150,9 @@
             return "check_string(space, %s)" % (value,)
         elif field.type in ("identifier",):
             if field.opt:
-                return "space.str_or_None_w(%s)" % (value,)
-            return "space.realstr_w(%s)" % (value,)
+                return ("space.realtext_w(%s) if not space.is_none(%s) "
+                        "else None" % (value, value))
+            return "space.realtext_w(%s)" % (value,)
         elif field.type in ("int",):
             return "space.int_w(%s)" % (value,)
         elif field.type in ("bool",):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1604,9 +1604,9 @@
         else:
             return buf.as_str()
 
-    def str_or_None_w(self, w_obj):
-        # YYY rename
-        return None if self.is_none(w_obj) else self.bytes_w(w_obj)
+    def text_or_none_w(self, w_obj):
+        # return text_w(w_obj) or None
+        return None if self.is_none(w_obj) else self.text_w(w_obj)
 
     def bytes_w(self, w_obj):
         "Takes a bytes object and returns an unwrapped RPython bytestring."
@@ -1617,15 +1617,11 @@
         unwrapped RPython bytestring."""
         return w_obj.str_w(self)
 
-    #@not_rpython    BACKCOMPAT: should be replaced with bytes_w or text_w
+    @not_rpython    # tests only; should be replaced with bytes_w or text_w
     def str_w(self, w_obj):
         """For tests only."""
         return self.bytes_w(w_obj)
 
-    #@not_rpython    BACKCOMPAT
-    def str0_w(self, w_obj):
-        return self.bytes0_w(w_obj)
-
     def bytes0_w(self, w_obj):
         "Like bytes_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
@@ -1647,6 +1643,9 @@
                                      getfilesystemencoding(self))
         return self.bytes0_w(w_obj)
 
+    def fsencode_or_none_w(self, w_obj):
+        return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1681,9 +1680,9 @@
         """
         return w_obj.float_w(self, allow_conversion)
 
-    def realstr_w(self, w_obj):
-        # YYY rename
-        # Like bytes_w, but only works if w_obj is really of type 'str'.
+    def realtext_w(self, w_obj):
+        # Like bytes_w(), but only works if w_obj is really of type 'str'.
+        # On Python 3 this is the same as text_w().
         if not self.isinstance_w(w_obj, self.w_bytes):
             raise oefmt(self.w_TypeError, "argument must be a string")
         return self.bytes_w(w_obj)
@@ -1702,8 +1701,8 @@
         return rstring.assert_str0(result)
 
     def realunicode_w(self, w_obj):
-        # Like unicode_w, but only works if w_obj is really of type
-        # 'unicode'.
+        # Like unicode_w(), but only works if w_obj is really of type
+        # 'unicode'.  On Python 3 this is the same as unicode_w().
         if not self.isinstance_w(w_obj, self.w_unicode):
             raise oefmt(self.w_TypeError, "argument must be a unicode")
         return self.unicode_w(w_obj)
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -145,10 +145,7 @@
     def visit_bufferstr(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
-    def visit_str_or_None(self, el, app_sig):
-        self.checked_space_method(el, app_sig)
-
-    def visit_str0(self, el, app_sig):
+    def visit_text_or_none(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
     def visit_bytes(self, el, app_sig):
@@ -166,6 +163,9 @@
     def visit_fsencode(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_fsencode_or_none(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -289,11 +289,8 @@
     def visit_bufferstr(self, typ):
         self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),))
 
-    def visit_str_or_None(self, typ):
-        self.run_args.append("space.str_or_None_w(%s)" % (self.scopenext(),))
-
-    def visit_str0(self, typ):
-        self.run_args.append("space.str0_w(%s)" % (self.scopenext(),))
+    def visit_text_or_none(self, typ):
+        self.run_args.append("space.text_or_none_w(%s)" % (self.scopenext(),))
 
     def visit_bytes(self, typ):
         self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
@@ -310,6 +307,9 @@
     def visit_fsencode(self, typ):
         self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
 
+    def visit_fsencode_or_none(self, typ):
+        self.run_args.append("space.fsencode_or_none_w(%s)" % (self.scopenext(),))
+
     def visit_nonnegint(self, typ):
         self.run_args.append("space.gateway_nonnegint_w(%s)" % (
             self.scopenext(),))
@@ -454,11 +454,8 @@
     def visit_bufferstr(self, typ):
         self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),))
 
-    def visit_str_or_None(self, typ):
-        self.unwrap.append("space.str_or_None_w(%s)" % (self.nextarg(),))
-
-    def visit_str0(self, typ):
-        self.unwrap.append("space.str0_w(%s)" % (self.nextarg(),))
+    def visit_text_or_none(self, typ):
+        self.unwrap.append("space.text_or_none_w(%s)" % (self.nextarg(),))
 
     def visit_bytes(self, typ):
         self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
@@ -475,6 +472,9 @@
     def visit_fsencode(self, typ):
         self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
 
+    def visit_fsencode_or_none(self, typ):
+        self.unwrap.append("space.fsencode_or_none_w(%s)" % (self.nextarg(),))
+
     def visit_nonnegint(self, typ):
         self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),))
 
@@ -606,6 +606,8 @@
                              "the name of an argument of the following "
                              "function" % (name,))
 
+    assert str not in unwrap_spec   # use 'text' or 'bytes' instead of str
+
     return unwrap_spec
 
 
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -18,7 +18,8 @@
 
 def compilecode(space, source, filename, cmd='exec'):
     w_code = space.builtin.call(
-        'compile', space.wrap(source), space.wrap(filename), space.wrap(cmd), space.newint(0), space.newint(0))
+        'compile', space.newtext(source), space.newtext(filename),
+        space.newtext(cmd), space.newint(0), space.newint(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
 
@@ -85,10 +86,11 @@
         argv.extend(args)
     space.setitem(space.sys.w_dict, space.newtext('argv'), space.wrap(argv))
     w_import = space.builtin.get('__import__')
-    runpy = space.call_function(w_import, space.wrap('runpy'))
-    w_run_module = space.getitem(runpy.w_dict, space.wrap('run_module'))
-    return space.call_function(w_run_module, space.wrap(module_name), space.w_None,
-                               space.wrap('__main__'), space.w_True)
+    runpy = space.call_function(w_import, space.newtext('runpy'))
+    w_run_module = space.getitem(runpy.w_dict, space.newtext('run_module'))
+    return space.call_function(w_run_module, space.newtext(module_name),
+                               space.w_None, space.newtext('__main__'),
+                               space.w_True)
 
 
 def run_toplevel(space, f, verbose=False):
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -48,7 +48,7 @@
             space.call_method(self.w_dict, 'update', self.w_initialdict)
 
         for w_submodule in self.submodules_w:
-            name = space.str0_w(w_submodule.w_name)
+            name = space.text0_w(w_submodule.w_name)
             space.setitem(self.w_dict, space.newtext(name.split(".")[-1]), w_submodule)
             space.getbuiltinmodule(name)
 
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -42,7 +42,7 @@
 
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
-        modulename = self.space.str0_w(self.w_name)
+        modulename = self.space.text0_w(self.w_name)
         self.space.builtin_modules[modulename] = self
 
     def setup_after_space_initialization(self):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -25,7 +25,7 @@
 
 # helper
 
-def unpack_text_tuple(space,w_str_tuple):
+def unpack_text_tuple(space, w_str_tuple):
     return [space.text_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -377,7 +377,7 @@
             return space.wrap(s0+s1)
         app_g3_ss = gateway.interp2app_temp(g3_ss,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      str, 'str_or_None'])
+                                                      'text', 'text_or_none'])
         w_app_g3_ss = space.wrap(app_g3_ss)
         assert self.space.eq_w(
             space.call(w_app_g3_ss,
@@ -512,7 +512,7 @@
 
         app_g3_s = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      str])
+                                                      'text'])
         w_app_g3_s = space.wrap(app_g3_s)
         assert space.eq_w(space.call_function(w_app_g3_s,w("foo")),w("foo"))
         raises(gateway.OperationError,space.call_function,w_app_g3_s,w(None))
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -212,11 +212,11 @@
         res = self.space.interp_w(Function, w(None), can_be_None=True)
         assert res is None
 
-    def test_str0_w(self):
+    def test_text0_w(self):
         space = self.space
         w = space.wrap
-        assert space.str0_w(w("123")) == "123"
-        exc = space.raises_w(space.w_TypeError, space.str0_w, w("123\x004"))
+        assert space.text0_w(w("123")) == "123"
+        exc = space.raises_w(space.w_TypeError, space.text0_w, w("123\x004"))
         assert space.unicode0_w(w(u"123")) == u"123"
         exc = space.raises_w(space.w_TypeError, space.unicode0_w, w(u"123\x004"))
 
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -16,17 +16,18 @@
                                              space.newtext(msg)]))
     return raise_unicode_exception_decode
 
-class RUnicodeEncodeError(Exception):
-    def __init__(self, encoding, object, start, end, reason):
-        self.encoding = encoding
-        self.object = object
-        self.start = start
-        self.end = end
-        self.reason = reason
-
-def raise_unicode_exception_encode(errors, encoding, msg, u,
-                                   startingpos, endingpos):
-    raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
+ at specialize.memo()
+def encode_error_handler(space):
+    # Fast version of the "strict" errors handler.
+    def raise_unicode_exception_encode(errors, encoding, msg, u,
+                                       startingpos, endingpos):
+        raise OperationError(space.w_UnicodeEncodeError,
+                             space.newtuple([space.newtext(encoding),
+                                             space.newunicode(u),
+                                             space.newint(startingpos),
+                                             space.newint(endingpos),
+                                             space.newtext(msg)]))
+    return raise_unicode_exception_encode
 
 # ____________________________________________________________
 
@@ -68,5 +69,5 @@
     # it stands for.  These are the Python2 rules; Python3 differs.
     return runicode.unicode_encode_utf_8(
         uni, len(uni), "strict",
-        errorhandler=raise_unicode_exception_encode,
+        errorhandler=None,
         allow_surrogates=True)
diff --git a/pypy/module/__pypy__/interp_os.py b/pypy/module/__pypy__/interp_os.py
--- a/pypy/module/__pypy__/interp_os.py
+++ b/pypy/module/__pypy__/interp_os.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 
 
- at unwrap_spec(name='str0')
+ at unwrap_spec(name='text0')
 def real_getenv(space, name):
     """Get an OS environment value skipping Python cache"""
     return space.newtext_or_none(os.environ.get(name))
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -572,7 +572,7 @@
         return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA)
 
 
-    @unwrap_spec(filename="str_or_None", flags=int)
+    @unwrap_spec(filename="fsencode_or_none", flags=int)
     def descr_dlopen(self, filename, flags=0):
         """\
 Load and return a dynamic library identified by 'name'.  The standard
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -91,7 +91,7 @@
 W_Library.typedef.acceptable_as_base_class = False
 
 
- at unwrap_spec(filename="str_or_None", flags=int)
+ at unwrap_spec(filename="fsencode_or_none", flags=int)
 def load_library(space, filename, flags=0):
     lib = W_Library(space, filename, flags)
     return lib
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -6,7 +6,7 @@
 import pypy.module.cpyext.api     # side-effect of pre-importing it
 
 
- at unwrap_spec(cdef=str, module_name=str, source=str, packed=int)
+ at unwrap_spec(cdef='text', module_name='text', source='text', packed=int)
 def prepare(space, cdef, module_name, source, w_includes=None,
             w_extra_source=None, w_min_version=None, packed=False):
     try:
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -314,12 +314,12 @@
     w_res = space.call_function(w_encoder, w_obj, space.newtext(errors))
     return space.getitem(w_res, space.newint(0))
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def readbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('s#', w_data)
     return space.newtuple([space.newbytes(s), space.newint(len(s))])
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def charbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('t#', w_data)
     return space.newtuple([space.newbytes(s), space.newint(len(s))])
@@ -373,7 +373,7 @@
 def make_encoder_wrapper(name):
     rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
     assert hasattr(runicode, rname)
-    @unwrap_spec(uni=unicode, errors='str_or_None')
+    @unwrap_spec(uni=unicode, errors='text_or_none')
     def wrap_encoder(space, uni, errors="strict"):
         if errors is None:
             errors = 'strict'
@@ -387,7 +387,7 @@
 def make_decoder_wrapper(name):
     rname = "str_decode_%s" % (name.replace("_decode", ""), )
     assert hasattr(runicode, rname)
-    @unwrap_spec(string='bufferstr', errors='str_or_None',
+    @unwrap_spec(string='bufferstr', errors='text_or_none',
                  w_final=WrappedDefault(False))
     def wrap_decoder(space, string, errors="strict", w_final=None):
         if errors is None:
@@ -437,7 +437,7 @@
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=True"
- at unwrap_spec(uni=unicode, errors='str_or_None')
+ at unwrap_spec(uni=unicode, errors='text_or_none')
 def utf_8_encode(space, uni, errors="strict"):
     if errors is None:
         errors = 'strict'
@@ -450,7 +450,7 @@
         allow_surrogates=True)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
- at unwrap_spec(string='bufferstr', errors='str_or_None',
+ at unwrap_spec(string='bufferstr', errors='text_or_none',
              w_final = WrappedDefault(False))
 def utf_8_decode(space, string, errors="strict", w_final=None):
     if errors is None:
@@ -466,7 +466,7 @@
         allow_surrogates=True)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
+ at unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
     if errors is None:
@@ -487,7 +487,7 @@
     return space.newtuple([space.newunicode(res), space.newint(consumed),
                            space.newint(byteorder)])
 
- at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
+ at unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_32_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
     final = space.is_true(w_final)
@@ -585,7 +585,7 @@
             "character mapping must return integer, None or str")
 
 
- at unwrap_spec(string='bufferstr', errors='str_or_None')
+ at unwrap_spec(string='bufferstr', errors='text_or_none')
 def charmap_decode(space, string, errors="strict", w_mapping=None):
     if errors is None:
         errors = 'strict'
@@ -604,7 +604,7 @@
         final, state.decode_error_handler, mapping)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(uni=unicode, errors='str_or_None')
+ at unwrap_spec(uni=unicode, errors='text_or_none')
 def charmap_encode(space, uni, errors="strict", w_mapping=None):
     if errors is None:
         errors = 'strict'
@@ -647,7 +647,7 @@
             return -1
         return space.int_w(w_code)
 
- at unwrap_spec(string='bufferstr', errors='str_or_None',
+ at unwrap_spec(string='bufferstr', errors='text_or_none',
              w_final=WrappedDefault(False))
 def unicode_escape_decode(space, string, errors="strict", w_final=None):
     if errors is None:
@@ -667,7 +667,7 @@
 # ____________________________________________________________
 # Unicode-internal
 
- at unwrap_spec(errors='str_or_None')
+ at unwrap_spec(errors='text_or_none')
 def unicode_internal_decode(space, w_string, errors="strict"):
     if errors is None:
         errors = 'strict'
@@ -691,7 +691,7 @@
 # support for the "string escape" codec
 # This is a bytes-to bytes transformation
 
- at unwrap_spec(data='bytes', errors='str_or_None')
+ at unwrap_spec(data='bytes', errors='text_or_none')
 def escape_encode(space, data, errors='strict'):
     from pypy.objspace.std.bytesobject import string_escape_encode
     result = string_escape_encode(data, quote="'")
@@ -701,7 +701,7 @@
     w_result = space.newbytes(result[start:end])
     return space.newtuple([w_result, space.newint(len(data))])
 
- at unwrap_spec(data='bytes', errors='str_or_None')
+ at unwrap_spec(data='bytes', errors='text_or_none')
 def escape_decode(space, data, errors='strict'):
     from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape
     result = PyString_DecodeEscape(space, data, errors, None)
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -666,7 +666,7 @@
     return False
 
 
- at unwrap_spec(w_file=W_File, encoding="str_or_None", errors="str_or_None")
+ at unwrap_spec(w_file=W_File, encoding="text_or_none", errors="text_or_none")
 def set_file_encoding(space, w_file, encoding=None, errors=None):
     w_file.encoding = encoding
     w_file.errors = errors
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -41,8 +41,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024
 
 @unwrap_spec(mode='text', buffering=int,
-             encoding="str_or_None", errors="str_or_None",
-             newline="str_or_None", closefd=bool)
+             encoding="text_or_none", errors="text_or_none",
+             newline="text_or_none", closefd=bool)
 def open(space, w_file, mode="r", buffering=-1, encoding=None, errors=None,
     newline=None, closefd=True):
     from pypy.module._io.interp_bufferedio import (W_BufferedRandom,
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -359,7 +359,7 @@
                                               # of the stream
         self.snapshot = None
 
-    @unwrap_spec(encoding="str_or_None", line_buffering=int)
+    @unwrap_spec(encoding="text_or_none", line_buffering=int)
     def descr_init(self, space, w_buffer, encoding=None,
                    w_errors=None, w_newline=None, line_buffering=0):
         self.state = STATE_ZERO
diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -68,7 +68,7 @@
         return space.newunicode(output)
 
 
- at unwrap_spec(errors="str_or_None")
+ at unwrap_spec(errors="text_or_none")
 def mbidecoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalDecoder, w_subtype)
     r.__init__(space, errors)
@@ -116,7 +116,7 @@
         return space.newbytes(output)
 
 
- at unwrap_spec(errors="str_or_None")
+ at unwrap_spec(errors="text_or_none")
 def mbiencoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalEncoder, w_subtype)
     r.__init__(space, errors)
diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py
--- a/pypy/module/_multibytecodec/interp_multibytecodec.py
+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py
@@ -11,7 +11,7 @@
         self.name = name
         self.codec = codec
 
-    @unwrap_spec(input='bytes', errors="str_or_None")
+    @unwrap_spec(input='bytes', errors="text_or_none")
     def decode(self, space, input, errors=None):
         if errors is None:
             errors = 'strict'
@@ -27,7 +27,7 @@
         return space.newtuple([space.newunicode(output),
                                space.newint(len(input))])
 
-    @unwrap_spec(input=unicode, errors="str_or_None")
+    @unwrap_spec(input=unicode, errors="text_or_none")
     def encode(self, space, input, errors=None):
         if errors is None:
             errors = 'strict'
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -114,7 +114,7 @@
 # __________________________________________________________
 # functions for the "win32" namespace
 
- at unwrap_spec(name=str, openmode=r_uint, pipemode=r_uint, maxinstances=r_uint,
+ at unwrap_spec(name='text', openmode=r_uint, pipemode=r_uint, maxinstances=r_uint,
              outputsize=r_uint, inputsize=r_uint, timeout=r_uint)
 def CreateNamedPipe(space, name, openmode, pipemode, maxinstances,
                     outputsize, inputsize, timeout, w_security):
@@ -161,13 +161,13 @@
         lltype.free(state, flavor='raw')
         lltype.free(statep, flavor='raw')
 
- at unwrap_spec(name=str, timeout=r_uint)
+ at unwrap_spec(name='text', timeout=r_uint)
 def WaitNamedPipe(space, name, timeout):
     # Careful: zero means "default value specified by CreateNamedPipe()"
     if not _WaitNamedPipe(name, timeout):
         raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
- at unwrap_spec(filename=str, access=r_uint, share=r_uint,
+ at unwrap_spec(filename='fsencode', access=r_uint, share=r_uint,
              disposition=r_uint, flags=r_uint)
 def CreateFile(space, filename, access, share, w_security,
                disposition, flags, w_templatefile):
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -344,7 +344,7 @@
     def getidentifier(self, space):
         return space.newint(self.cdll.getidentifier())
 
- at unwrap_spec(name='str_or_None', mode=int)
+ at unwrap_spec(name='fsencode_or_none', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
     return W_CDLL(space, name, mode)
 
@@ -363,7 +363,7 @@
         W_CDLL.__init__(self, space, name, mode)
         self.flags = libffi.FUNCFLAG_STDCALL
 
- at unwrap_spec(name='str_or_None', mode=int)
+ at unwrap_spec(name='fsencode_or_none', mode=int)
 def descr_new_windll(space, w_type, name, mode=-1):
     return W_WinDLL(space, name, mode)
 
diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py
--- a/pypy/module/_rawffi/alt/test/test_struct.py
+++ b/pypy/module/_rawffi/alt/test/test_struct.py
@@ -43,7 +43,7 @@
     def setup_class(cls):
         BaseAppTestFFI.setup_class.im_func(cls)
 
-        @unwrap_spec(addr=int, typename=str, length=int)
+        @unwrap_spec(addr=int, typename='text', length=int)
         def read_raw_mem(space, addr, typename, length):
             import ctypes
             addr = ctypes.cast(addr, ctypes.c_void_p)
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -235,7 +235,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(name='str_or_None')
+ at unwrap_spec(name='fsencode_or_none')
 def descr_new_cdll(space, w_type, name):
     cdll = open_cdll(space, name)
     return W_CDLL(space, name, cdll)
diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -99,7 +99,7 @@
                     usage.c_rgpszUsageIdentifier[i]))
             return space.newset(result_w)
 
- at unwrap_spec(store_name=str)
+ at unwrap_spec(store_name='text')
 def enum_certificates_w(space, store_name):
     """enum_certificates(store_name) -> []
 
@@ -142,7 +142,7 @@
 
     return space.newlist(result_w)
 
- at unwrap_spec(store_name=str)
+ at unwrap_spec(store_name='text')
 def enum_crls_w(space, store_name):
     """enum_crls(store_name) -> []
 
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -689,7 +689,7 @@
 
 The return value is the handle of the opened key.
 If the function fails, an EnvironmentError exception is raised."""
-    machine = space.str_or_None_w(w_machine)
+    machine = space.text_or_none_w(w_machine)
     hkey = hkey_w(w_hkey, space)
     with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
         ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -328,7 +328,7 @@
         raise oefmt(space.w_ValueError, "cannot open in read-write mode")
     if basemode == "a":
         raise oefmt(space.w_ValueError, "cannot append to bz2 file")
-    stream = open_path_helper(space.str0_w(w_path), os_flags, False)
+    stream = open_path_helper(space.fsencode_w(w_path), os_flags, False)
     if reading:
         bz2stream = ReadBZ2Filter(space, stream, buffering)
         buffering = 0     # by construction, the ReadBZ2Filter acts like
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -28,7 +28,7 @@
         data = DATA[:100]
         f.write(data, 'wb')
 
-    @unwrap_spec(data=str)
+    @unwrap_spec(data='bytes')
     def decompress(space, data):
         import popen2
         import bz2
diff --git a/pypy/module/cppyy/test/test_crossing.py b/pypy/module/cppyy/test/test_crossing.py
--- a/pypy/module/cppyy/test/test_crossing.py
+++ b/pypy/module/cppyy/test/test_crossing.py
@@ -78,7 +78,7 @@
             import ctypes, cppyy""")    # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
-        @unwrap_spec(name=str, init=str, body=str)
+        @unwrap_spec(name='text', init='text', body='text')
         def create_cdll(space, name, init, body):
             # the following is loosely from test_cpyext.py import_module; it
             # is copied here to be able to tweak the call to
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -119,7 +119,7 @@
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     except ValueError:
-        if not space.isinstance_w(w_obj, space.w_str):
+        if not space.isinstance_w(w_obj, space.w_bytes):
             # XXX Python 3?
             raise BufferError("could not create buffer from object")
         view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.bytes_w(w_obj), track_allocation=False))
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -237,7 +237,10 @@
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
     check_num_args(space, w_args, 0)
-    return space.newint(generic_cpy_call(space, func_len, w_self))
+    res = generic_cpy_call(space, func_len, w_self)
+    if widen(res) == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.newint(res)
 
 def wrap_sq_item(space, w_self, w_args, func):
     func_target = rffi.cast(ssizeargfunc, func)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -287,7 +287,7 @@
         if self.runappdirect:
             return
 
-        @unwrap_spec(name=str)
+        @unwrap_spec(name='text')
         def compile_module(space, name,
                            w_source_files=None,
                            w_source_strings=None):
@@ -313,8 +313,8 @@
 
             return space.wrap(pydname)
 
-        @unwrap_spec(name=str, init='str_or_None', body=str,
-                     filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+        @unwrap_spec(name='text', init='text_or_none', body='text',
+                     filename='fsencode_or_none', PY_SSIZE_T_CLEAN=bool)
         def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
@@ -325,12 +325,12 @@
             return w_result
 
 
-        @unwrap_spec(mod=str, name=str)
+        @unwrap_spec(mod='text', name='text')
         def load_module(space, mod, name):
             return self.sys_info.load_module(mod, name)
 
-        @unwrap_spec(modname=str, prologue=str,
-                             more_init=str, PY_SSIZE_T_CLEAN=bool)
+        @unwrap_spec(modname='text', prologue='text',
+                             more_init='text', PY_SSIZE_T_CLEAN=bool)
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -178,7 +178,6 @@
             ("bounce", "METH_NOARGS",
             """
             PyGILState_STATE gilstate;
-            PyThreadState *tstate;
             PyObject *dict;
 
             if (PyEval_ThreadsInitialized() == 0)
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -288,7 +288,7 @@
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_object = w_object
         self.w_start = w_start
@@ -628,11 +628,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.realstr_w(w_encoding)
-        space.realstr_w(w_object)
+        space.realtext_w(w_encoding)
+        space.realtext_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
@@ -718,11 +718,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.realstr_w(w_encoding)
+        space.realtext_w(w_encoding)
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.realtext_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -79,7 +79,7 @@
 
 # ____________________________________________________________
 
- at unwrap_spec(filename='str0')
+ at unwrap_spec(filename='fsencode')
 def dump_heap_stats(space, filename):
     tb = rgc._heap_stats()
     if not tb:
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -364,7 +364,7 @@
         length = space.len_w(w_fromlist)
         for i in range(length):
             w_name = space.getitem(w_fromlist, space.newint(i))
-            if not space.isinstance_w(w_name, space.w_str):
+            if not space.isinstance_w(w_name, space.w_text):
                 raise oefmt(space.w_TypeError,
                     "'fromlist' items must be str, not %T", w_name)
         if w_path is not None:
@@ -491,7 +491,7 @@
     def __init__(self, space):
         pass
 
-    @unwrap_spec(path='str0')
+    @unwrap_spec(path='fsencode')
     def descr_init(self, space, path):
         if not path:
             raise oefmt(space.w_ImportError, "empty pathname")
@@ -570,7 +570,7 @@
                 if w_loader:
                     return FindInfo.fromLoader(w_loader)
 
-            path = space.str0_w(w_pathitem)
+            path = space.fsencode_w(w_pathitem)
             filepart = os.path.join(path, partname)
             log_pyverbose(space, 2, "# trying %s\n" % (filepart,))
             if os.path.isdir(filepart) and case_ok(filepart):
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -78,7 +78,7 @@
 def load_module(space, w_name, w_file, w_filename, w_info):
     w_suffix, w_filemode, w_modtype = space.unpackiterable(w_info, 3)
 
-    filename = space.str0_w(w_filename)
+    filename = space.fsencode_w(w_filename)
     filemode = space.text_w(w_filemode)
     if space.is_w(w_file, space.w_None):
         stream = None
@@ -95,7 +95,7 @@
         space, w_name, find_info, reuse=True)
 
 def load_source(space, w_modulename, w_filename, w_file=None):
-    filename = space.str0_w(w_filename)
+    filename = space.fsencode_w(w_filename)
 
     stream = get_file(space, w_file, filename, 'U')
 
@@ -109,7 +109,7 @@
         stream.close()
     return w_mod
 
- at unwrap_spec(filename='str0', check_afterwards=int)
+ at unwrap_spec(filename='fsencode', check_afterwards=int)
 def _run_compiled_module(space, w_modulename, filename, w_file, w_module,
                          check_afterwards=False):
     # the function 'imp._run_compiled_module' is a pypy-only extension
@@ -125,14 +125,14 @@
         stream.close()
     return w_mod
 
- at unwrap_spec(filename='str0')
+ at unwrap_spec(filename='fsencode')
 def load_compiled(space, w_modulename, filename, w_file=None):
     w_mod = Module(space, w_modulename)
     importing._prepare_module(space, w_mod, filename, None)
     return _run_compiled_module(space, w_modulename, filename, w_file, w_mod,
                                 check_afterwards=True)
 
- at unwrap_spec(filename='text')
+ at unwrap_spec(filename='fsencode')
 def load_dynamic(space, w_modulename, filename, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -183,7 +183,7 @@
 
     def descr_setitem(self, w_index, w_value):
         space = self.space
-        value = space.realstr_w(w_value)
+        value = space.realtext_w(w_value)
         self.check_valid()
 
         self.check_writeable()
@@ -238,7 +238,7 @@
         if not space.isinstance_w(w_item, space.w_bytes):
             raise oefmt(space.w_IndexError,
                         "mmap slice assignment must be a string")
-        value = space.realstr_w(w_item)
+        value = space.realtext_w(w_item)
         if len(value) != (j - i):
             raise oefmt(space.w_IndexError,
                         "mmap slice assignment is wrong size")
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -147,17 +147,17 @@
         return None
 
 
- at unwrap_spec(executable='str0')
+ at unwrap_spec(executable='fsencode')
 def pypy_find_executable(space, executable):
     return space.newtext(find_executable(executable))
 
 
- at unwrap_spec(filename='str0')
+ at unwrap_spec(filename='fsencode')
 def pypy_resolvedirof(space, filename):
     return space.newtext(resolvedirof(filename))
 
 
- at unwrap_spec(executable='str0')
+ at unwrap_spec(executable='fsencode')
 def pypy_find_stdlib(space, executable):
     path, prefix = None, None
     if executable != '*':
diff --git a/pypy/module/test_lib_pypy/test_md5_extra.py b/pypy/module/test_lib_pypy/test_md5_extra.py
--- a/pypy/module/test_lib_pypy/test_md5_extra.py
+++ b/pypy/module/test_lib_pypy/test_md5_extra.py
@@ -94,7 +94,7 @@
             # interp2app doesn't work in appdirect mode
             cls.w_compare_host = staticmethod(compare_host)
         else:
-            compare_host.unwrap_spec = [str, str, str]
+            compare_host.unwrap_spec = ['bytes', 'bytes', 'text']
             cls.w_compare_host = space.wrap(gateway.interp2app(compare_host))
 
     def w_compare(self, message):
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -350,7 +350,7 @@
         space = self.space
         return space.newtext(self.filename)
 
- at unwrap_spec(name='str0')
+ at unwrap_spec(name='text0')
 def descr_new_zipimporter(space, w_type, name):
     ok = False
     parts_ends = [i for i in range(0, len(name))
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -45,7 +45,6 @@
 
     def str_w(self, space):
         return NonConstant("foobar")
-    identifier_w = bytes_w = str_w
 
     def unicode_w(self, space):
         return NonConstant(u"foobar")
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -181,7 +181,7 @@
         # we ignore w_type and always return a bytearray
         return new_bytearray(space, space.w_bytearray, data)
 
-    @unwrap_spec(encoding='str_or_None', errors='str_or_None')
+    @unwrap_spec(encoding='text_or_none', errors='text_or_none')
     def descr_init(self, space, w_source=None, encoding=None, errors=None):
         if w_source is None:
             w_source = space.newbytes('')
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -89,12 +89,15 @@
         for typedef, cls in builtin_type_classes.items():
             w_type = self.gettypeobject(typedef)
             self.builtin_types[typedef.name] = w_type
-            if 1: # typedef.name != "str":      BACKCOMPAT
-                setattr(self, 'w_' + typedef.name, w_type)
-            if typedef.name == "str":
-                self.w_bytes = w_type
+            name = typedef.name
+            # we don't expose 'space.w_str' at all, to avoid confusion
+            # with Python 3.  Instead, in Python 2, it becomes
+            # space.w_bytes (or space.w_text).
+            if name == 'str':
+                name = 'bytes'
+            setattr(self, 'w_' + name, w_type)
             self._interplevel_classes[w_type] = cls
-        self.w_text = self.w_bytes # this is w_unicode on Py3
+        self.w_text = self.w_bytes   # 'space.w_text' is w_unicode on Py3
         self.w_dict.flag_map_or_seq = 'M'
         self.builtin_types["NotImplemented"] = self.w_NotImplemented
         self.builtin_types["Ellipsis"] = self.w_Ellipsis
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -899,7 +899,7 @@
                 successes = entry.success_counter
             globalfailures = INVALID_CACHE_ENTRY.failure_counter
             return space.wrap((failures, successes, globalfailures))
-        check.unwrap_spec = [gateway.ObjSpace, gateway.W_Root, str]
+        check.unwrap_spec = [gateway.ObjSpace, gateway.W_Root, 'text']
         cls.w_check = cls.space.wrap(gateway.interp2app(check))
 
     def test_simple(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -462,26 +462,17 @@
         w_encoder = space.sys.get_w_default_encoder()
     else:
         if errors is None or errors == 'strict':
-            try:
-                if encoding == 'ascii':
-                    u = space.unicode_w(w_object)
-                    eh = unicodehelper.raise_unicode_exception_encode
-                    return space.newbytes(unicode_encode_ascii(
-                            u, len(u), None, errorhandler=eh))
-                if encoding == 'utf-8':
-                    u = space.unicode_w(w_object)
-                    eh = unicodehelper.raise_unicode_exception_encode
-                    return space.newbytes(unicode_encode_utf_8(
-                            u, len(u), None, errorhandler=eh,
-                            allow_surrogates=True))
-            except unicodehelper.RUnicodeEncodeError as ue:
-                raise OperationError(space.w_UnicodeEncodeError,
-                                     space.newtuple([
-                    space.newtext(ue.encoding),
-                    space.newunicode(ue.object),
-                    space.newint(ue.start),
-                    space.newint(ue.end),
-                    space.newtext(ue.reason)]))
+            if encoding == 'ascii':
+                u = space.unicode_w(w_object)
+                eh = unicodehelper.encode_error_handler(space)
+                return space.newbytes(unicode_encode_ascii(
+                        u, len(u), None, errorhandler=eh))
+            if encoding == 'utf-8':
+                u = space.unicode_w(w_object)
+                eh = unicodehelper.encode_error_handler(space)
+                return space.newbytes(unicode_encode_utf_8(
+                        u, len(u), None, errorhandler=eh,
+                        allow_surrogates=True))
         from pypy.module._codecs.interp_codecs import lookup_codec
         w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
     if errors is None:
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -5,7 +5,7 @@
 from rpython.rlib.unicodedata import unicodedb
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import jit
+from rpython.rlib import jit, nonconst
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -133,6 +133,26 @@
 def _invalid_cont_byte(ordch):
     return ordch>>6 != 0x2    # 0b10
 
+_invalid_byte_2_of_2 = _invalid_cont_byte
+_invalid_byte_3_of_3 = _invalid_cont_byte
+_invalid_byte_3_of_4 = _invalid_cont_byte
+_invalid_byte_4_of_4 = _invalid_cont_byte
+
+ at enforceargs(allow_surrogates=bool)
+def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xe0 and ordch2 < 0xa0)
+            # surrogates shouldn't be valid UTF-8!
+            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
+
+def _invalid_byte_2_of_4(ordch1, ordch2):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xf0 and ordch2 < 0x90) or
+            (ordch1 == 0xf4 and ordch2 > 0x8f))
+
+# NOTE: this is a slightly fixed algorithm when compared with
+# CPython2's.  It is closer to CPython3's.  See comments in
+# test_invalid_cb_for_3bytes_seq().
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates):
     if size == 0:
@@ -153,20 +173,60 @@
         if pos + n > size:
             if not final:
                 break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
             charsleft = size - pos - 1 # either 0, 1, 2
             # note: when we get the 'unexpected end of data' we need
             # to care about the pos returned; it can be lower than size,
             # in case we need to continue running this loop
-            endpos = pos + 1
-            if charsleft >= 1 and not _invalid_cont_byte(ord(s[pos+1])):
-                endpos = pos + 2
-                if charsleft >= 2 and not _invalid_cont_byte(ord(s[pos+2])):
-                    endpos = pos + 3
-            r, pos = errorhandler(errors, 'utf8',
-                                  'unexpected end of data',
-                                  s, pos, endpos)
-            result.append(r)
-            continue
+            if not charsleft:
+                # there's only the start byte and nothing else
+                r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+1)
+                result.append(r)
+                continue
+            ordch2 = ord(s[pos+1])
+            if n == 3:
+                # 3-bytes seq with only a continuation byte
+                if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+                    # second byte invalid, take the first and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                else:
+                    # second byte valid, but third byte missing
+                    r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+2)
+                    result.append(r)
+                    continue
+            elif n == 4:
+                # 4-bytes seq with 1 or 2 continuation bytes
+                if _invalid_byte_2_of_4(ordch1, ordch2):
+                    # second byte invalid, take the first and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+1)
+                    result.append(r)
+                    continue
+                elif charsleft == 2 and _invalid_byte_3_of_4(ord(s[pos+2])):
+                    # third byte invalid, take the first two and continue
+                    r, pos = errorhandler(errors, 'utf8',
+                                          'invalid continuation byte',
+                                          s, pos, pos+2)
+                    result.append(r)
+                    continue
+                else:
+                    # there's only 1 or 2 valid cb, but the others are missing
+                    r, pos = errorhandler(errors, 'utf8',
+                                      'unexpected end of data',
+                                      s, pos, pos+charsleft+1)
+                    result.append(r)
+                    continue
+            raise AssertionError("unreachable")
 
         if n == 0:
             r, pos = errorhandler(errors, 'utf8',
@@ -179,7 +239,7 @@
 
         elif n == 2:
             ordch2 = ord(s[pos+1])
-            if _invalid_cont_byte(ordch2):
+            if _invalid_byte_2_of_2(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
@@ -193,48 +253,41 @@
         elif n == 3:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
-            if _invalid_cont_byte(ordch2):
+            if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif _invalid_cont_byte(ordch3):
+            elif _invalid_byte_3_of_3(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
             # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
-            c = (((ordch1 & 0x0F) << 12) +     # 0b00001111
-                 ((ordch2 & 0x3F) << 6) +      # 0b00111111
-                 (ordch3 & 0x3F))              # 0b00111111
-            if c < 2048 or (0xd800 <= c <= 0xdfff and not allow_surrogates):
-                r, pos = errorhandler(errors, 'utf8',
-                                      'invalid continuation byte',
-                                      s, pos, pos+2)
-                result.append(r)
-                continue
-            result.append(unichr(c))
+            result.append(unichr(((ordch1 & 0x0F) << 12) +     # 0b00001111
+                                 ((ordch2 & 0x3F) << 6) +      # 0b00111111
+                                 (ordch3 & 0x3F)))             # 0b00111111
             pos += 3
 
         elif n == 4:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
             ordch4 = ord(s[pos+3])
-            if _invalid_cont_byte(ordch2):
+            if _invalid_byte_2_of_4(ordch1, ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif _invalid_cont_byte(ordch3):
+            elif _invalid_byte_3_of_4(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
-            elif _invalid_cont_byte(ordch4):
+            elif _invalid_byte_4_of_4(ordch4):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+3)
@@ -245,12 +298,6 @@
                  ((ordch2 & 0x3F) << 12) +      # 0b00111111
                  ((ordch3 & 0x3F) << 6) +       # 0b00111111
                  (ordch4 & 0x3F))               # 0b00111111
-            if c <= 65535 or c > 0x10ffff:
-                r, pos = errorhandler(errors, 'utf8',
-                                      'invalid continuation byte',
-                                      s, pos, pos+3)
-                result.append(r)
-                continue
             if c <= MAXUNICODE:
                 result.append(UNICHR(c))
             else:
@@ -326,7 +373,12 @@
                             pos += 1
                             _encodeUCS4(result, ch3)
                             continue
-                    if not allow_surrogates:
+                    # note: if the program only ever calls this with
+                    # allow_surrogates=True, then we'll never annotate
+                    # the following block of code, and errorhandler()
+                    # will never be called.  This causes RPython
+                    # problems.  Avoid it with the nonconst hack.
+                    if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
                                                    s, pos-1, pos)
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -700,27 +700,6 @@
             assert decoder(seq, len(seq), 'ignore', final=True
                            ) == (res, len(seq))
 
-    @settings(max_examples=10000)
-    @given(strategies.binary())
-    def test_str_check_utf8(self, s):
-        try:
-            u = s.decode("utf8")
-            valid = True
-        except UnicodeDecodeError as e:
-            valid = False
-        try:
-            result, length = runicode.str_decode_utf_8(s, len(s), None,
-                errorhandler=None, final=True, allow_surrogates=True)
-        except UnicodeDecodeError as a:
-            assert not valid
-            assert a.start == e.start
-            assert a.end == e.end
-            assert str(a) == str(e)
-        else:
-            assert valid
-            assert result == u
-            assert length == len(s)
-
 
 class TestEncoding(UnicodeTests):
     def test_all_ascii(self):
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1733,7 +1733,11 @@
                      (ulimitv, ' '.join(args),)]
             popen = subprocess.Popen(args1, stderr=subprocess.PIPE)
             _, child_stderr = popen.communicate()
-            assert popen.wait() == 134     # aborted
+            assert popen.wait() in (-6, 134)     # aborted
+            # note: it seems that on some systems we get 134 and on
+            # others we get -6.  Bash is supposed to translate the
+            # SIGABRT (signal 6) from the subprocess into the exit 
+            # code 128+6, but I guess it may not always do so.
             assert 'out of memory:' in child_stderr
             return '42'
         #

From pypy.commits at gmail.com  Fri Feb 24 02:42:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Feb 2017 23:42:01 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58afe3c9.554c190a.3a96d.8b8e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90337:6d6e4f94cf81
Date: 2017-02-24 08:41 +0100
http://bitbucket.org/pypy/pypy/changeset/6d6e4f94cf81/

Log:	merge heads

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -74,7 +74,7 @@
 ^rpython/doc/_build/.*$
 ^compiled
 ^.git/
-^.hypothesis/
+.hypothesis/
 ^release/
 ^rpython/_cache$
 
diff --git a/extra_tests/README.txt b/extra_tests/README.txt
new file mode 100644
--- /dev/null
+++ b/extra_tests/README.txt
@@ -0,0 +1,5 @@
+The tests in this directory are a complement to lib-python/3/test/.
+
+They are meant to run on top of a compiled pypy3 or CPython3.5 in an
+environment containing at least pytest and hypothesis, using a command like
+'pytest extra_tests/'.
diff --git a/extra_tests/pytest.ini b/extra_tests/pytest.ini
new file mode 100644
diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_unicode.py
@@ -0,0 +1,34 @@
+import pytest
+from hypothesis import strategies as st
+from hypothesis import given, settings, example
+
+from unicodedata import normalize
+
+# For every (n1, n2, n3) triple, applying n1 then n2 must be the same
+# as applying n3.
+# Reference: http://unicode.org/reports/tr15/#Design_Goals
+compositions = [
+    ('NFC', 'NFC', 'NFC'),
+    ('NFC', 'NFD', 'NFD'),
+    ('NFC', 'NFKC', 'NFKC'),
+    ('NFC', 'NFKD', 'NFKD'),
+    ('NFD', 'NFC', 'NFC'),
+    ('NFD', 'NFD', 'NFD'),
+    ('NFD', 'NFKC', 'NFKC'),
+    ('NFD', 'NFKD', 'NFKD'),
+    ('NFKC', 'NFC', 'NFKC'),
+    ('NFKC', 'NFD', 'NFKD'),
+    ('NFKC', 'NFKC', 'NFKC'),
+    ('NFKC', 'NFKD', 'NFKD'),
+    ('NFKD', 'NFC', 'NFKC'),
+    ('NFKD', 'NFD', 'NFKD'),
+    ('NFKD', 'NFKC', 'NFKC'),
+    ('NFKD', 'NFKD', 'NFKD'),
+]
+
+ at pytest.mark.parametrize('norm1, norm2, norm3', compositions)
+ at settings(max_examples=1000)
+ at example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at given(s=st.text())
+def test_composition(s, norm1, norm2, norm3):
+    assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s)

From pypy.commits at gmail.com  Fri Feb 24 04:42:22 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 24 Feb 2017 01:42:22 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: add stefan beyer to the list
Message-ID: <58affffe.8fdb190a.16705.94cc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5790:8faac664baf4
Date: 2017-02-24 10:42 +0100
http://bitbucket.org/pypy/extradoc/changeset/8faac664baf4/

Log:	add stefan beyer to the list

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -17,6 +17,7 @@
 Ronan Lamy           25.02/04.03    Ermina
 Antonio Cuni         27.02/03.03    Ermina
 Maciej Fijalkowski   26.02/01.03    Ermina
+Stefan Beyer         26.02/04.03    Ermina
 ==================== ============== =======================
 
 **NOTE:** lodging is by default in Ermina.  There are two ~4 people

From pypy.commits at gmail.com  Fri Feb 24 09:00:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 06:00:42 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: Move offline the 'nextpage'
 pointer from each page
Message-ID: <58b03c8a.8e48190a.65b34.9bae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90338:42e9fc22ba08
Date: 2017-02-24 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/42e9fc22ba08/

Log:	Move offline the 'nextpage' pointer from each page

diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -62,14 +62,6 @@
 
 PAGE_PTR = lltype.Ptr(lltype.ForwardReference())
 PAGE_HEADER = lltype.Struct('PageHeader',
-    # -- The following pointer makes a chained list of pages.  For non-full
-    #    pages, it is a chained list of pages having the same size class,
-    #    rooted in 'page_for_size[size_class]'.  For full pages, it is a
-    #    different chained list rooted in 'full_page_for_size[size_class]'.
-    #    For free pages, it is the list 'freepages' in the arena header.
-    ('nextpage', PAGE_PTR),
-    # -- The arena this page is part of.
-    ('arena', ARENA_PTR),
     # -- The number of free blocks.  The numbers of uninitialized and
     #    allocated blocks can be deduced from the context if needed.
     ('nfree', lltype.Signed),
@@ -77,11 +69,24 @@
     #    first uninitialized block (pointing to data that is uninitialized,
     #    or to the end of the page).
     ('freeblock', llmemory.Address),
+    # -- The following pointer makes a chained list of pages.  For non-full
+    #    pages, it is a chained list of pages having the same size class,
+    #    rooted in 'page_for_size[size_class]'.  For full pages, it is a
+    #    different chained list rooted in 'full_page_for_size[size_class]'.
+    #    For free pages, it is the list 'freepages' in the arena header.
+    # == NOTE: with offline_visited_flags, this pointer contains an
+    #    indirection.  It is only written once, and so it must not be
+    #    the first word of the page, because that is overwritten with
+    #    a chained list when the page becomes free again.
+    ('nextpage', llmemory.Address),
+    # -- The arena this page is part of.
+    ('arena', ARENA_PTR),
     # -- The structure above is 4 words, which is a good value:
     #    '(1024-4) % N' is zero or very small for various small N's,
     #    i.e. there is not much wasted space.
     )
 PAGE_PTR.TO.become(PAGE_HEADER)
+PAGE_PTR_PTR = rffi.CArrayPtr(PAGE_PTR)
 PAGE_NULL = lltype.nullptr(PAGE_HEADER)
 
 # ----------
@@ -153,6 +158,13 @@
         # the additional bookkeeping stuff.
         self.total_memory_used = r_uint(0)
 
+        if not offline_visited_flags:
+            self.ARENA_WITH_OFFL = ARENA
+        else:
+            self.ARENA_WITH_OFFL = lltype.Struct('ArenaReferenceOffl',
+                ('arena_super', ARENA),
+                ('nextindirections', lltype.Array(PAGE_PTR)))
+
 
     def _new_page_ptr_list(self, length):
         return lltype.malloc(rffi.CArray(PAGE_PTR), length,
@@ -195,8 +207,8 @@
         if freeblock - pageaddr > self.page_size - nsize:
             # This was the last free block, so unlink the page from the
             # chained list and put it in the 'full_page_for_size' list.
-            self.page_for_size[size_class] = page.nextpage
-            page.nextpage = self.full_page_for_size[size_class]
+            self.page_for_size[size_class] = self.getnextpage(page)
+            self.setnextpage(page, self.full_page_for_size[size_class])
             self.full_page_for_size[size_class] = page
         #
         llarena.arena_reserve(result, _dummy_size(size))
@@ -227,6 +239,17 @@
             ll_assert(self.num_uninitialized_pages > 0,
                       "fully allocated arena found in self.current_arena")
             self.num_uninitialized_pages -= 1
+            #
+            if self.offline_visited_flags:
+                # initialize the 'page.nextpage' indirection
+                arena_with_offl = lltype.cast_pointer(
+                    lltype.Ptr(self.ARENA_WITH_OFFL), arena)
+                array = arena_with_offl.nextindirections
+                array = lltype.direct_arrayitems(array)
+                p = lltype.direct_ptradd(array, self.num_uninitialized_pages)
+                page = llmemory.cast_adr_to_ptr(result, PAGE_PTR)
+                page.nextpage = llmemory.cast_ptr_to_adr(p)
+            #
             if self.num_uninitialized_pages > 0:
                 freepages = result + self.page_size
             else:
@@ -248,7 +271,7 @@
         page.arena = arena
         page.nfree = 0
         page.freeblock = result + self.hdrsize
-        page.nextpage = PAGE_NULL
+        self.setnextpage(page, PAGE_NULL)
         ll_assert(self.page_for_size[size_class] == PAGE_NULL,
                   "allocate_new_page() called but a page is already waiting")
         self.page_for_size[size_class] = page
@@ -327,7 +350,12 @@
             arena_base.arena._from_minimarkpage = True
         #
         # Allocate an ARENA object and initialize it
-        arena = lltype.malloc(ARENA, flavor='raw', track_allocation=False)
+        if not self.offline_visited_flags:
+            arena = lltype.malloc(ARENA, flavor='raw', track_allocation=False)
+        else:
+            arena_with_offl = lltype.malloc(self.ARENA_WITH_OFFL, npages,
+                                         flavor='raw', track_allocation=False)
+            arena = arena_with_offl.arena_super
         arena.base = arena_base
         arena.nfreepages = 0        # they are all uninitialized pages
         arena.totalpages = npages
@@ -458,7 +486,7 @@
                 #
                 # Collect the page.
                 surviving = self.walk_page(page, block_size, ok_to_free_func)
-                nextpage = page.nextpage
+                nextpage = self.getnextpage(page)
                 #
                 if surviving == nblocks:
                     #
@@ -466,14 +494,14 @@
                     # 'remaining_full_pages' chained list.
                     ll_assert(step == 0,
                               "A non-full page became full while freeing")
-                    page.nextpage = remaining_full_pages
+                    self.setnextpage(page, remaining_full_pages)
                     remaining_full_pages = page
                     #
                 elif surviving > 0:
                     #
                     # There is at least 1 object surviving.  Re-insert
                     # the page in the 'remaining_partial_pages' chained list.
-                    page.nextpage = remaining_partial_pages
+                    self.setnextpage(page, remaining_partial_pages)
                     remaining_partial_pages = page
                     #
                 else:
@@ -619,6 +647,21 @@
         return getattr(arena, '_from_minimarkpage', False)
 
 
+    @always_inline
+    def getnextpage(self, page):
+        if not self.offline_visited_flags:
+            return llmemory.cast_adr_to_ptr(page.nextpage, PAGE_PTR)
+        else:
+            return llmemory.cast_adr_to_ptr(page.nextpage, PAGE_PTR_PTR)[0]
+
+    @always_inline
+    def setnextpage(self, page, nextp):
+        if not self.offline_visited_flags:
+            page.nextpage = llmemory.cast_ptr_to_adr(nextp)
+        else:
+            llmemory.cast_adr_to_ptr(page.nextpage, PAGE_PTR_PTR)[0] = nextp
+
+
     @staticmethod
     @always_inline
     def get_64bit_ptr_visited_flags(obj):
diff --git a/rpython/memory/gc/test/test_minimarkpage.py b/rpython/memory/gc/test/test_minimarkpage.py
--- a/rpython/memory/gc/test/test_minimarkpage.py
+++ b/rpython/memory/gc/test/test_minimarkpage.py
@@ -38,7 +38,7 @@
         assert page.nfree == 0
         page1 = page.freeblock - hdrsize
         assert llmemory.cast_ptr_to_adr(page) == page1
-        assert page.nextpage == PAGE_NULL
+        assert ac.getnextpage(page) == PAGE_NULL
     #
     ac = ArenaCollection(arenasize, pagesize, 99)
     assert ac.num_uninitialized_pages == 0
@@ -83,7 +83,7 @@
             chainedlists = ac.page_for_size
         else:
             chainedlists = ac.full_page_for_size
-        page.nextpage = chainedlists[size_class]
+        ac.setnextpage(page, chainedlists[size_class])
         page.arena = ac.current_arena
         chainedlists[size_class] = page
         if fill_with_objects:
@@ -285,7 +285,7 @@
                                hdrsize + 2*WORD: False}
     page = getpage(ac, 0)
     assert page == ac.page_for_size[2]
-    assert page.nextpage == PAGE_NULL
+    assert ac.getnextpage(page) == PAGE_NULL
     assert ac._nuninitialized(page, 2) == 1
     assert page.nfree == 0
     chkob(ac, 0, 4*WORD, page.freeblock)
@@ -313,7 +313,7 @@
                                hdrsize + 4*WORD: False}
     page = getpage(ac, 0)
     assert page == ac.full_page_for_size[2]
-    assert page.nextpage == PAGE_NULL
+    assert ac.getnextpage(page) == PAGE_NULL
     assert ac._nuninitialized(page, 2) == 0
     assert page.nfree == 0
     assert freepages(ac) == NULL
@@ -331,7 +331,7 @@
     page = getpage(ac, 0)
     pageaddr = pagenum(ac, 0)
     assert page == ac.page_for_size[2]
-    assert page.nextpage == PAGE_NULL
+    assert ac.getnextpage(page) == PAGE_NULL
     assert ac._nuninitialized(page, 2) == 0
     assert page.nfree == 2
     assert page.freeblock == pageaddr + hdrsize + 2*WORD
@@ -356,7 +356,7 @@
     page = getpage(ac, 0)
     pageaddr = pagenum(ac, 0)
     assert page == ac.page_for_size[2]
-    assert page.nextpage == PAGE_NULL
+    assert ac.getnextpage(page) == PAGE_NULL
     assert ac._nuninitialized(page, 2) == 4
     assert page.nfree == 4
     assert page.freeblock == pageaddr + hdrsize + 2*WORD
@@ -384,7 +384,7 @@
     page = getpage(ac, 0)
     pageaddr = pagenum(ac, 0)
     assert page == ac.page_for_size[2]
-    assert page.nextpage == PAGE_NULL
+    assert ac.getnextpage(page) == PAGE_NULL
     assert ac._nuninitialized(page, 2) == 4
     assert page.nfree == 6
     fb = page.freeblock

From pypy.commits at gmail.com  Fri Feb 24 10:19:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 07:19:55 -0800 (PST)
Subject: [pypy-commit] pypy gc-forkfriendly-2: fix accidental unsharing
Message-ID: <58b04f1b.02aa190a.4df23.9f95@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-forkfriendly-2
Changeset: r90339:bdef0258a588
Date: 2017-02-24 15:19 +0000
http://bitbucket.org/pypy/pypy/changeset/bdef0258a588/

Log:	fix accidental unsharing

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1254,15 +1254,17 @@
     @specialize.arg(2)
     def set_visited_and_other_flag(self, obj, other_flag):
         # Optimized 'set_visited(obj)' followed by setting 'other_flag' too
-        tid = self.header(obj).tid | other_flag
-        must_set_gcflag_visited = True
+        tid = self.header(obj).tid
+        add = GCFLAG_VISITED | other_flag
         if self.offline_visited_flags:
             if tid & GCFLAG_OLD_OUTSIDE_MINIMARKPAGE == 0:
                 self.ac.set_visited(obj)
-                must_set_gcflag_visited = False
-        if must_set_gcflag_visited:
-            tid |= GCFLAG_VISITED
-        self.header(obj).tid = tid
+                if tid & other_flag:
+                    return   # common case: 'other_flag' is already set
+                             # (not just an optimization: avoids writes
+                             # to this object, and thus fork-unsharing)
+                add = other_flag
+        self.header(obj).tid = tid | add
 
     def clear_visited(self, obj):
         # Clear the object's GCFLAG_VISITED flag, noting that it may be

From pypy.commits at gmail.com  Fri Feb 24 10:39:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 07:39:58 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2484: detail
Message-ID: <58b053ce.97212e0a.a38fd.693e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90340:06c69aa49c64
Date: 2017-02-24 16:39 +0100
http://bitbucket.org/pypy/pypy/changeset/06c69aa49c64/

Log:	Issue #2484: detail

diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -312,7 +312,7 @@
 
 } PyTypeObject;
 
-typedef struct {
+typedef struct _heaptypeobject {
     PyTypeObject ht_type;
     PyNumberMethods as_number;
     PyMappingMethods as_mapping;

From pypy.commits at gmail.com  Fri Feb 24 11:09:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 08:09:49 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2483: use ``#include ".."``
 instead of ``#include <..>`` to refer to
Message-ID: <58b05acd.cd092e0a.85d09.7d45@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90341:82c823854add
Date: 2017-02-24 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/82c823854add/

Log:	Issue #2483: use ``#include ".."`` instead of ``#include <..>`` to
	refer to a file in the same directory

diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -73,7 +73,7 @@
 
 #define Py_MEMCPY memcpy
 
-#include <pypy_macros.h>
+#include "pypy_macros.h"
 
 #include "patchlevel.h"
 #include "pyconfig.h"
@@ -137,7 +137,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-  #include <pypy_decl.h>
+  #include "pypy_decl.h"
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-#include <cpyext_object.h>
+#include "cpyext_object.h"
 
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -5,7 +5,7 @@
 extern "C" {
 #endif
 
-#include <cpyext_unicodeobject.h>
+#include "cpyext_unicodeobject.h"
 
 #ifdef __cplusplus
 }

From pypy.commits at gmail.com  Fri Feb 24 14:00:11 2017
From: pypy.commits at gmail.com (fijal)
Date: Fri, 24 Feb 2017 11:00:11 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: (fijal,
 arigo) kill helpers methods that don't belong to rutf8
Message-ID: <58b082bb.4a98190a.44098.a979@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90342:4caeffafa080
Date: 2017-02-24 19:59 +0100
http://bitbucket.org/pypy/pypy/changeset/4caeffafa080/

Log:	(fijal, arigo) kill helpers methods that don't belong to rutf8

	They also don't belong to runicode, have a more minimal change for
	now, but should be fixed at some point

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -39,18 +39,20 @@
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
     # XXX pick better length, maybe
-    result, consumed, length = rutf8.str_decode_utf8_escape(
+    # XXX that guy does not belong in runicode (nor in rutf8)
+    result_u, consumed = runicode.str_decode_unicode_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         unicodedata_handler=unicodedata_handler)
-    return result, length
+    return result_u.encode('utf8'), len(result_u)
 
 def decode_raw_unicode_escape(space, string):
     # XXX pick better length, maybe
-    result, consumed, length = rutf8.str_decode_raw_utf8_escape(
+    # XXX that guy does not belong in runicode (nor in rutf8)
+    result_u, consumed = runicode.str_decode_raw_unicode_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space))
-    return result, length
+    return result_u.encode('utf8'), len(result_u)
 
 def check_utf8(space, string):
     # Surrogates are accepted and not treated specially at all.
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -1,6 +1,7 @@
 
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib import runicode, jit
+from rpython.rlib.rarithmetic import r_uint
 from rpython.rlib.nonconst import NonConstant
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -29,6 +30,36 @@
                 chr((0x80 | (code & 0x3f)))), lgt
     raise ValueError
 
+def unichr_as_utf8_append(builder, code):
+    """ Encode code (numeric value) as utf8 encoded string
+    """
+    if code < 0:
+        raise ValueError
+    lgt = 1
+    if code >= runicode.MAXUNICODE:
+        lgt = 2
+    if code < 0x80:
+        # Encode ASCII
+        builder.append(chr(code))
+        return 1
+    if code < 0x0800:
+        # Encode Latin-1
+        builder.append(chr((0xc0 | (code >> 6))))
+        builder.append(chr((0x80 | (code & 0x3f))))
+        return lgt
+    if code < 0x10000:
+        builder.append(chr((0xe0 | (code >> 12))))
+        builder.append(chr((0x80 | ((code >> 6) & 0x3f))))
+        builder.append(chr((0x80 | (code & 0x3f))))
+        return lgt
+    if code < 0x10ffff:
+        builder.append(chr((0xf0 | (code >> 18))))
+        builder.append(chr((0x80 | ((code >> 12) & 0x3f))))
+        builder.append(chr((0x80 | ((code >> 6) & 0x3f))))
+        builder.append(chr((0x80 | (code & 0x3f))))
+        return lgt
+    raise ValueError
+
 def next_codepoint_pos(code, pos):
     """ Gives the position of the next codepoint after pos, -1
     if it's the last one (assumes valid utf8)
@@ -50,6 +81,13 @@
 def default_unicode_error_check(*args):
     xxx
 
+def default_unicode_error_decode(errors, encoding, message, s, pos, endpos, lgt):
+    if errors == 'replace':
+        return '\xef\xbf\xbd', endpos, lgt + 1 # u'\ufffd'
+    if errors == 'ignore':
+        return '', endpos, lgt
+    raise UnicodeDecodeError(encoding, s, pos, endpos, message)
+
 def check_newline_utf8(s, pos):
     chr1 = ord(s[pos])
     if 0xa <= chr1 <= 0xd:
@@ -207,184 +245,3 @@
     return pos, lgt
 str_check_utf8_elidable = jit.elidable(
     func_with_new_name(str_check_utf8_impl, "str_check_utf8_elidable"))
-
-
-def str_decode_raw_utf8_escape(s, size, errors, final=False,
-                               errorhandler=None):
-    lgt = 0
-    if errorhandler is None:
-        errorhandler = None # default_unicode_error_decode
-    if size == 0:
-        return '', 0, 0
-    result = StringBuilder(size)
-    pos = 0
-    while pos < size:
-        ch = s[pos]
-
-        # Non-escape characters are interpreted as Unicode ordinals
-        if ch != '\\':
-            result.append(ch)
-            pos += 1
-            lgt += 1
-            continue
-
-        # \u-escapes are only interpreted iff the number of leading
-        # backslashes is odd
-        bs = pos
-        while pos < size:
-            pos += 1
-            if pos == size or s[pos] != '\\':
-                break
-            lgt += 1
-            result.append('\\')
-
-        # we have a backslash at the end of the string, stop here
-        if pos >= size:
-            lgt += 1
-            result.append('\\')
-            break
-
-        if ((pos - bs) & 1 == 0 or
-            pos >= size or
-            (s[pos] != 'u' and s[pos] != 'U')):
-            result.append('\\')
-            result.append(s[pos])
-            lgt += 2
-            pos += 1
-            continue
-
-        digits = 4 if s[pos] == 'u' else 8
-        message = "truncated \\uXXXX"
-        pos += 1
-        xxx # change hexescape to deal with utf8
-        pos = hexescape(result, s, pos, digits,
-                        "rawunicodeescape", errorhandler, message, errors)
-
-    return result.build(), pos, lgt
-
-def str_decode_utf8_escape(s, size, errors, final=False,
-                              errorhandler=None,
-                              unicodedata_handler=None):
-    if errorhandler is None:
-        errorhandler = default_unicode_error_decode
-
-    if size == 0:
-        return '', 0, 0
-
-    lgt = 0
-    builder = StringBuilder(size)
-    pos = 0
-    while pos < size:
-        ch = s[pos]
-
-        # Non-escape characters are interpreted as Unicode ordinals
-        if ch != '\\':
-            builder.append(ch)
-            pos += 1
-            lgt += 1
-            continue
-
-        # - Escapes
-        pos += 1
-        if pos >= size:
-            message = "\\ at end of string"
-            res, pos = errorhandler(errors, "unicodeescape",
-                                    message, s, pos-1, size)
-            builder.append(res)
-            lgt += 1
-            continue
-
-        ch = s[pos]
-        pos += 1
-        # \x escapes
-        if ch == '\n': pass
-        elif ch == '\\': builder.append('\\'); lgt += 1
-        elif ch == '\'': builder.append('\''); lgt += 1
-        elif ch == '\"': builder.append('\"'); lgt += 1
-        elif ch == 'b' : builder.append('\b'); lgt += 1
-        elif ch == 'f' : builder.append('\f'); lgt += 1
-        elif ch == 't' : builder.append('\t'); lgt += 1
-        elif ch == 'n' : builder.append('\n'); lgt += 1
-        elif ch == 'r' : builder.append('\r'); lgt += 1
-        elif ch == 'v' : builder.append('\v'); lgt += 1
-        elif ch == 'a' : builder.append('\a'); lgt += 1
-        elif '0' <= ch <= '7':
-            xxx
-            x = ord(ch) - ord('0')
-            if pos < size:
-                ch = s[pos]
-                if '0' <= ch <= '7':
-                    pos += 1
-                    x = (x<<3) + ord(ch) - ord('0')
-                    if pos < size:
-                        ch = s[pos]
-                        if '0' <= ch <= '7':
-                            pos += 1
-                            x = (x<<3) + ord(ch) - ord('0')
-            builder.append(unichr(x))
-        # hex escapes
-        # \xXX
-        elif ch == 'x':
-            xxx
-            digits = 2
-            message = "truncated \\xXX escape"
-            pos = hexescape(builder, s, pos, digits,
-                            "unicodeescape", errorhandler, message, errors)
-
-        # \uXXXX
-        elif ch == 'u':
-            xxx
-            digits = 4
-            message = "truncated \\uXXXX escape"
-            pos = hexescape(builder, s, pos, digits,
-                            "unicodeescape", errorhandler, message, errors)
-
-        #  \UXXXXXXXX
-        elif ch == 'U':
-            xxx
-            digits = 8
-            message = "truncated \\UXXXXXXXX escape"
-            pos = hexescape(builder, s, pos, digits,
-                            "unicodeescape", errorhandler, message, errors)
-
-        # \N{name}
-        elif ch == 'N' and unicodedata_handler is not None:
-            xxx
-            message = "malformed \\N character escape"
-            look = pos
-
-            if look < size and s[look] == '{':
-                # look for the closing brace
-                while look < size and s[look] != '}':
-                    look += 1
-                if look < size and s[look] == '}':
-                    # found a name.  look it up in the unicode database
-                    message = "unknown Unicode character name"
-                    name = s[pos+1:look]
-                    code = unicodedata_handler.call(name)
-                    if code < 0:
-                        res, pos = errorhandler(errors, "unicodeescape",
-                                                message, s, pos-1, look+1)
-                        builder.append(res)
-                        continue
-                    pos = look + 1
-                    if code <= MAXUNICODE:
-                        builder.append(UNICHR(code))
-                    else:
-                        code -= 0x10000L
-                        builder.append(unichr(0xD800 + (code >> 10)))
-                        builder.append(unichr(0xDC00 + (code & 0x03FF)))
-                else:
-                    res, pos = errorhandler(errors, "unicodeescape",
-                                            message, s, pos-1, look+1)
-                    builder.append(res)
-            else:
-                res, pos = errorhandler(errors, "unicodeescape",
-                                        message, s, pos-1, look+1)
-                builder.append(res)
-        else:
-            builder.append('\\')
-            builder.append(ch)
-            lgt += 2
-
-    return builder.build(), pos, lgt
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -47,11 +47,6 @@
         assert consumed == len(s)
         assert length == len(u)
 
- at given(strategies.binary())
-def test_str_decode_raw_utf8_escape(uni):
-    return # XXX fix details
-    rutf8.str_decode_raw_utf8_escape(uni, len(uni), None)
-
 @given(strategies.characters())
 def test_next_pos(uni):
     skips = []

From pypy.commits at gmail.com  Fri Feb 24 16:37:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 13:37:47 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: update
Message-ID: <58b0a7ab.1a4b2e0a.b0c4d.693e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5791:bb6b773e92fb
Date: 2017-02-24 22:37 +0100
http://bitbucket.org/pypy/extradoc/changeset/bb6b773e92fb/

Log:	update

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -13,7 +13,7 @@
 Richard Plangger     26.02/04.03    Ermina
 Remi Meier           27.02/04.03    Ermina
 Manuel Jacob         25.02/04.03    Ermina
-Robert Zaremba       26?.02/01.03   Ermina
+Robert Zaremba       26.02/01.03    Ermina
 Ronan Lamy           25.02/04.03    Ermina
 Antonio Cuni         27.02/03.03    Ermina
 Maciej Fijalkowski   26.02/01.03    Ermina

From pypy.commits at gmail.com  Sat Feb 25 01:41:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 22:41:04 -0800 (PST)
Subject: [pypy-commit] pypy default: There are three includes of
 npy_common.h from three files in the same
Message-ID: <58b12700.09532e0a.2ab89.736b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90343:3a53ec66f961
Date: 2017-02-25 07:27 +0100
http://bitbucket.org/pypy/pypy/changeset/3a53ec66f961/

Log:	There are three includes of npy_common.h from three files in the
	same directory. They use three different styles, at least one of
	which is wrong according to issue #2483. Fix.

diff --git a/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h b/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
--- a/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
+++ b/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
@@ -1,7 +1,7 @@
 #ifndef NDARRAYTYPES_H
 #define NDARRAYTYPES_H
 
-#include "numpy/npy_common.h"
+#include "npy_common.h"
 //#include "npy_endian.h"
 //#include "npy_cpu.h"
 //#include "utils.h"
diff --git a/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
@@ -10,7 +10,7 @@
 #ifndef _NPY_3KCOMPAT_H_
 #define _NPY_3KCOMPAT_H_
 
-#include <numpy/npy_common.h>
+#include "npy_common.h"
 
 #define npy_PyFile_Dup(file, mode) (NULL)
 #define npy_PyFile_DupClose(file, handle) (0)

From pypy.commits at gmail.com  Sat Feb 25 01:41:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 22:41:06 -0800 (PST)
Subject: [pypy-commit] pypy default: a 'temporary fix' seems to become
 permanent, document
Message-ID: <58b12702.04162e0a.21a12.7350@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90344:dbf7a1921b61
Date: 2017-02-25 07:40 +0100
http://bitbucket.org/pypy/pypy/changeset/dbf7a1921b61/

Log:	a 'temporary fix' seems to become permanent, document

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1235,6 +1235,17 @@
 
 def write_header(header_name, decls):
     lines = [
+        '''
+#ifdef _WIN64
+/* this check is for sanity, but also because the 'temporary fix'
+   below seems to become permanent and would cause unexpected
+   nonsense on Win64---but note that it's not the only reason for
+   why Win64 is not supported!  If you want to help, see
+   http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+   */
+#  error "PyPy does not support 64-bit on Windows.  Use Win32"
+#endif
+''',
         '#define Signed   long           /* xxx temporary fix */',
         '#define Unsigned unsigned long  /* xxx temporary fix */',
         '',] + decls + [

From pypy.commits at gmail.com  Sat Feb 25 02:13:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 23:13:55 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58b12eb3.16502e0a.696c.684c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r865:8908314f0d9c
Date: 2017-02-25 08:13 +0100
http://bitbucket.org/pypy/pypy.org/changeset/8908314f0d9c/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66648 of $105000 (63.5%)
+   $66667 of $105000 (63.5%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2469 left
+    we have $2487 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sat Feb 25 02:31:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 23:31:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fixes
Message-ID: <58b132c7.88012e0a.da919.c642@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90345:4ff73331b9e8
Date: 2017-02-25 08:30 +0100
http://bitbucket.org/pypy/pypy/changeset/4ff73331b9e8/

Log:	fixes

diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -50,7 +50,7 @@
         if cls.runappdirect:
             cls.w_decompress = decompress
         else:
-            @gateway.unwrap_spec(data=bytes)
+            @gateway.unwrap_spec(data='bytes')
             def decompress_w(space, data):
                 return space.newbytes(decompress(cls, data))
             cls.w_decompress = cls.space.wrap(gateway.interp2app(decompress_w))
@@ -228,7 +228,7 @@
         if cls.runappdirect:
             cls.w_decompress = decompress
         else:
-            @gateway.unwrap_spec(data=bytes)
+            @gateway.unwrap_spec(data='bytes')
             def decompress_w(space, data):
                 return space.newbytes(decompress(cls, data))
             cls.w_decompress = cls.space.wrap(gateway.interp2app(decompress_w))
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -208,8 +208,8 @@
     __length_hint__=interp2app(W_UnpackIter.descr_length_hint)
 )
 
- at unwrap_spec(format='text')
-def iter_unpack(space, format, w_buffer):
+def iter_unpack(space, w_format, w_buffer):
+    format = text_or_bytes_w(space, w_format)
     w_struct = W_Struct(space, format)
     return W_UnpackIter(space, w_struct, w_buffer)
 

From pypy.commits at gmail.com  Sat Feb 25 02:34:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Feb 2017 23:34:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix another test for 4d3783a2d4a2
Message-ID: <58b1337c.014b2e0a.3338c.6e23@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90346:f8aeb679acc9
Date: 2017-02-25 08:33 +0100
http://bitbucket.org/pypy/pypy/changeset/f8aeb679acc9/

Log:	fix another test for 4d3783a2d4a2

diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -149,11 +149,12 @@
         assert self.s("{0:d}").format(G("data")) == "G(data)"
         assert self.s("{0!s}").format(G("data")) == "string is data"
 
-        msg = "non-empty format string passed to object.__format__"
+        msg = "unsupported format string passed to E.__format__"
         e = raises(TypeError, self.s("{0:^10}").format, E("data"))
         assert str(e.value) == msg
         e = raises(TypeError, self.s("{0:^10s}").format, E("data"))
         assert str(e.value) == msg
+        msg = "unsupported format string passed to G.__format__"
         e = raises(TypeError, self.s("{0:>15s}").format, G("data"))
         assert str(e.value) == msg
 

From pypy.commits at gmail.com  Sat Feb 25 03:22:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 25 Feb 2017 00:22:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: remove a new_interned_str() that has no
 equivalent in pypy2 and that probably makes kwargsdict slow
Message-ID: <58b13ebd.0a482e0a.d0ed2.76b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90347:e0d25df618e7
Date: 2017-02-25 09:21 +0100
http://bitbucket.org/pypy/pypy/changeset/e0d25df618e7/

Log:	remove a new_interned_str() that has no equivalent in pypy2 and that
	probably makes kwargsdict slow

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -11,7 +11,7 @@
 
 
 def _wrapkey(space, key):
-    return space.new_interned_str(key)
+    return space.newtext(key)
 
 
 class EmptyKwargsDictStrategy(EmptyDictStrategy):

From pypy.commits at gmail.com  Sat Feb 25 04:03:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 25 Feb 2017 01:03:33 -0800 (PST)
Subject: [pypy-commit] pypy default: I think we no longer need the
 'create_ops' flag. It was added as
Message-ID: <58b14865.501d190a.4bf14.c3e1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90348:da6be6540744
Date: 2017-02-25 10:03 +0100
http://bitbucket.org/pypy/pypy/changeset/da6be6540744/

Log:	I think we no longer need the 'create_ops' flag. It was added as
	replacement for a limitation of the previous state of the world, but
	there is no reason to keep that logic. Added a test that shows
	better results without it (it's a case I've seen in pypy, and the
	removed 'strlen' means a guard afterwards will be dropped)

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4840,6 +4840,21 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_nonvirtual_newstr_strlen(self):
+        ops = """
+        [p0]
+        p1 = call_r(0, p0, s"X", descr=strconcatdescr)
+        i0 = strlen(p1)
+        finish(i0)
+        """
+        expected = """
+        [p0]
+        i2 = strlen(p0)
+        i4 = int_add(i2, 1)
+        finish(i4)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
     def test_copy_long_string_to_virtual(self):
         ops = """
         []
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -60,7 +60,6 @@
         self.length = length
         self._is_virtual = is_virtual
         self.mode = mode
-        self.length = length
 
     def getlenbound(self, mode):
         from rpython.jit.metainterp.optimizeopt import intutils
@@ -110,12 +109,11 @@
         return self.string_copy_parts(op, string_optimizer, targetbox,
                                       offsetbox, mode)
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         if self.lgtop is not None:
             return self.lgtop
         assert not self.is_virtual()
-        if not create_ops:
-            return None
         lengthop = ResOperation(mode.STRLEN, [op])
         lengthop.set_forwarded(self.getlenbound(mode))
         self.lgtop = lengthop
@@ -172,7 +170,8 @@
     def is_virtual(self):
         return self._is_virtual
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         if self.lgtop is None:
             self.lgtop = ConstInt(len(self._chars))
         return self.lgtop
@@ -251,7 +250,8 @@
             return s1[start : start + length]
         return None
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         return self.lgtop
 
     def _visitor_walk_recursive(self, instbox, visitor, optimizer):
@@ -280,20 +280,19 @@
     def is_virtual(self):
         return self._is_virtual
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         if self.lgtop is not None:
             return self.lgtop
         lefti = string_optimizer.getptrinfo(self.vleft)
-        len1box = lefti.getstrlen(self.vleft, string_optimizer, mode,
-                                  create_ops)
+        len1box = lefti.getstrlen(self.vleft, string_optimizer, mode)
         if len1box is None:
             return None
         righti = string_optimizer.getptrinfo(self.vright)
-        len2box = righti.getstrlen(self.vright, string_optimizer, mode,
-                                   create_ops)
+        len2box = righti.getstrlen(self.vright, string_optimizer, mode)
         if len2box is None:
             return None
-        self.lgtop = _int_add(string_optimizer, len1box, len2box, create_ops)
+        self.lgtop = _int_add(string_optimizer, len1box, len2box)
             # ^^^ may still be None, if string_optimizer is None
         return self.lgtop
 
@@ -371,7 +370,7 @@
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(string_optimizer, box1, box2, create_ops=True):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -379,8 +378,6 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if not create_ops:
-        return None
     op = ResOperation(rop.INT_ADD, [box1, box2])
     string_optimizer.send_extra_operation(op)
     return op
@@ -529,9 +526,10 @@
         return self._optimize_STRLEN(op, mode_unicode)
 
     def _optimize_STRLEN(self, op, mode):
-        opinfo = self.getptrinfo(op.getarg(0))
+        arg1 = self.get_box_replacement(op.getarg(0))
+        opinfo = self.getptrinfo(arg1)
         if opinfo:
-            lgtop = opinfo.getstrlen(op, self, mode, False)
+            lgtop = opinfo.getstrlen(arg1, self, mode)
             if lgtop is not None:
                 self.make_equal_to(op, lgtop)
                 return
@@ -700,11 +698,11 @@
         i2 = self.getptrinfo(arg2)
         #
         if i1:
-            l1box = i1.getstrlen(arg1, self, mode, create_ops=False)
+            l1box = i1.getstrlen(arg1, self, mode)
         else:
             l1box = None
         if i2:
-            l2box = i2.getstrlen(arg2, self, mode, create_ops=False)
+            l2box = i2.getstrlen(arg2, self, mode)
         else:
             l2box = None
         if (l1box is not None and l2box is not None and
@@ -742,7 +740,7 @@
         l2box = None
         l1box = None
         if i2:
-            l2box = i2.getstrlen(arg2, self, mode, create_ops=False)
+            l2box = i2.getstrlen(arg2, self, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
                 if i1 and i1.is_nonnull():
@@ -760,7 +758,7 @@
                     return True, None
             if l2box.value == 1:
                 if i1:
-                    l1box = i1.getstrlen(arg1, self, mode, False)
+                    l1box = i1.getstrlen(arg1, self, mode)
                 if isinstance(l1box, ConstInt) and l1box.value == 1:
                     # comparing two single chars
                     vchar1 = self.strgetitem(None, arg1, optimizer.CONST_0,
@@ -799,7 +797,7 @@
         i2 = self.getptrinfo(arg2)
         l2box = None
         if i2:
-            l2box = i2.getstrlen(arg1, self, mode, create_ops=False)
+            l2box = i2.getstrlen(arg1, self, mode)
         if l2box:
             l2info = self.getintbound(l2box)
             if l2info.is_constant():
@@ -822,12 +820,14 @@
         return False, None
 
     def opt_call_stroruni_STR_CMP(self, op, mode):
-        i1 = self.getptrinfo(op.getarg(1))
-        i2 = self.getptrinfo(op.getarg(2))
+        arg1 = self.get_box_replacement(op.getarg(1))
+        arg2 = self.get_box_replacement(op.getarg(2))
+        i1 = self.getptrinfo(arg1)
+        i2 = self.getptrinfo(arg2)
         if not i1 or not i2:
             return False, None
-        l1box = i1.getstrlen(None, self, mode, False)
-        l2box = i2.getstrlen(None, self, mode, False)
+        l1box = i1.getstrlen(arg1, self, mode)
+        l2box = i2.getstrlen(arg2, self, mode)
         if (l1box is not None and l2box is not None and
             isinstance(l1box, ConstInt) and
             isinstance(l2box, ConstInt) and

From pypy.commits at gmail.com  Sat Feb 25 05:32:44 2017
From: pypy.commits at gmail.com (fijal)
Date: Sat, 25 Feb 2017 02:32:44 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: go a slightly shorter way to
 working system - use decode a bit everywhere
Message-ID: <58b15d4c.84472e0a.5e5fd.7f2b@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90349:6cb0a61d37f2
Date: 2017-02-25 11:32 +0100
http://bitbucket.org/pypy/pypy/changeset/6cb0a61d37f2/

Log:	go a slightly shorter way to working system - use decode a bit
	everywhere

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -155,7 +155,7 @@
         if isinstance(x, str):
             return self.newtext(x)
         if isinstance(x, unicode):
-            return self.newunicode(x)
+            return self.newutf8(x.encode('utf8'), len(x))
         if isinstance(x, float):
             return W_FloatObject(x)
         if isinstance(x, W_Root):
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -22,6 +22,7 @@
         assert len(warnings) == 2
 
     def test_listview_unicode(self):
+        py.test.skip("skip for new")
         w_str = self.space.wrap(u'abcd')
         assert self.space.listview_unicode(w_str) == list(u"abcd")
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -110,10 +110,11 @@
                          "found", len(self._value))
         return space.newint(ord(self._value[0]))
 
-    def _new(self, value, length):
-        return W_UnicodeObject(value, length)
+    def _new(self, value):
+        return W_UnicodeObject(value.encode('utf8', len(value)))
 
     def _new_from_list(self, value):
+        xxx
         return W_UnicodeObject(u''.join(value))
 
     def _empty(self):
@@ -124,7 +125,8 @@
             self._length = self._compute_length()
         return self._length
 
-    _val = utf8_w
+    def _val(self, space):
+        return self._utf8.decode('utf8')
 
     @staticmethod
     def _use_rstr_ops(space, w_other):
@@ -134,17 +136,10 @@
 
     @staticmethod
     def _op_val(space, w_other, strict=None):
-        if isinstance(w_other, W_UnicodeObject):
-            return w_other._utf8
-        if space.isinstance_w(w_other, space.w_bytes):
-            return unicode_from_string(space, w_other)._utf8
-        if strict:
-            raise oefmt(space.w_TypeError,
-                "%s arg must be None, unicode or str", strict)
-        return unicode_from_encoded_object(
-            space, w_other, None, "strict")._utf8
+        return W_UnicodeObject._convert_to_unicode(space, w_other)._utf8.decode('utf8')
 
-    def _convert_to_unicode(self, space, w_other):
+    @staticmethod
+    def _convert_to_unicode(space, w_other):
         if isinstance(w_other, W_UnicodeObject):
             return w_other
         if space.isinstance_w(w_other, space.w_bytes):
@@ -240,7 +235,7 @@
         return w_newobj
 
     def descr_repr(self, space):
-        chars = self._value
+        chars = self._utf8.decode('utf8')
         size = len(chars)
         s = _repr_function(chars, size, "strict")
         return space.newtext(s)
@@ -254,7 +249,7 @@
 
     def descr_eq(self, space, w_other):
         try:
-            res = self._val(space) == self._op_val(space, w_other)
+            res = self._utf8 == self._convert_to_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -270,7 +265,7 @@
 
     def descr_ne(self, space, w_other):
         try:
-            res = self._val(space) != self._op_val(space, w_other)
+            res = self._utf8 != self._convert_to_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -286,7 +281,7 @@
 
     def descr_lt(self, space, w_other):
         try:
-            res = self._val(space) < self._op_val(space, w_other)
+            res = self._utf8 < self._convert_to_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -295,7 +290,7 @@
 
     def descr_le(self, space, w_other):
         try:
-            res = self._val(space) <= self._op_val(space, w_other)
+            res = self._utf8 <= self._convert_to_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -304,7 +299,7 @@
 
     def descr_gt(self, space, w_other):
         try:
-            res = self._val(space) > self._op_val(space, w_other)
+            res = self._utf8 > self._convert_to_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -313,7 +308,7 @@
 
     def descr_ge(self, space, w_other):
         try:
-            res = self._val(space) >= self._op_val(space, w_other)
+            res = self._utf8 >= self._convert_to_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -339,6 +334,7 @@
         return mod_format(space, w_values, self, do_unicode=True)
 
     def descr_translate(self, space, w_table):
+        xxx
         selfvalue = self._value
         w_sys = space.getbuiltinmodule('sys')
         maxunicode = space.int_w(space.getattr(w_sys,
@@ -379,9 +375,10 @@
     def descr_join(self, space, w_list):
         l = space.listview_unicode(w_list)
         if l is not None:
+            xxx
             if len(l) == 1:
                 return space.newunicode(l[0])
-            return space.newunicode(self._val(space).join(l))
+            return space.newunicode(self._utf8).join(l)
         return self._StringMethods_descr_join(space, w_list)
 
     def _join_return_one(self, space, w_obj):
@@ -471,11 +468,11 @@
             if value and i != 0:
                 sb.append(value)
             sb.append(unwrapped[i])
-        return self._new(sb.build(), lgt)
+        return W_UnicodeObject(sb.build(), lgt)
 
     @unwrap_spec(keepends=bool)
     def descr_splitlines(self, space, keepends=False):
-        value = self._val(space)
+        value = self._utf8
         length = len(value)
         strs_w = []
         pos = 0
@@ -501,13 +498,13 @@
 
     @unwrap_spec(width=int)
     def descr_zfill(self, space, width):
-        selfval = self._val(space)
+        selfval = self._utf8
         if len(selfval) == 0:
-            return self._new(self._multi_chr(self._chr('0')) * width, width)
+            return W_UnicodeObject(self._multi_chr(self._chr('0')) * width, width)
         num_zeros = width - self._len()
         if num_zeros <= 0:
             # cannot return self, in case it is a subclass of str
-            return self._new(selfval, self._len())
+            return W_UnicodeObject(selfval, self._len())
         builder = self._builder(num_zeros + len(selfval))
         if len(selfval) > 0 and (selfval[0] == '+' or selfval[0] == '-'):
             # copy sign to first position
@@ -517,18 +514,18 @@
             start = 0
         builder.append_multiple_char(self._chr('0'), num_zeros)
         builder.append_slice(selfval, start, len(selfval))
-        return self._new(builder.build(), width)
+        return W_UnicodeObject(builder.build(), width)
 
     @unwrap_spec(maxsplit=int)
     def descr_split(self, space, w_sep=None, maxsplit=-1):
         # XXX maybe optimize?
         res = []
-        value = self._val(space)
+        value = self._utf8
         if space.is_none(w_sep):
             res = split(value, maxsplit=maxsplit)
             return space.newlist([W_UnicodeObject(s, -1) for s in res])
 
-        by = self._op_val(space, w_sep)
+        by = self._convert_to_unicode(space, w_sep)._utf8
         if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = split(value, by, maxsplit)
@@ -538,18 +535,40 @@
     @unwrap_spec(maxsplit=int)
     def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
         res = []
-        value = self._val(space)
+        value = self._utf8
         if space.is_none(w_sep):
             res = rsplit(value, maxsplit=maxsplit)
             return space.newlist([W_UnicodeObject(s, -1) for s in res])
 
-        by = self._op_val(space, w_sep)
+        by = self._convert_to_unicode(space, w_sep)._utf8
         if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = rsplit(value, by, maxsplit)
 
         return space.newlist([W_UnicodeObject(s, -1) for s in res])
 
+    @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
+    def descr_center(self, space, width, w_fillchar):
+        value = self._utf8
+        fillchar = self._convert_to_unicode(space, w_fillchar)._utf8
+        if len(fillchar) != 1:
+            raise oefmt(space.w_TypeError,
+                        "center() argument 2 must be a single character")
+
+        d = width - self._len()
+        if d > 0:
+            offset = d//2 + (d & width & 1)
+            fillchar = self._multi_chr(fillchar[0])
+            centered = offset * fillchar + value + (d - offset) * fillchar
+        else:
+            centered = value
+            d = 0
+
+        return W_UnicodeObject(centered, self._len() + d)
+
+    def descr_title(self, space):
+        return 
+
 def wrapunicode(space, uni):
     return W_UnicodeObject(uni)
 

From pypy.commits at gmail.com  Sat Feb 25 08:05:55 2017
From: pypy.commits at gmail.com (fijal)
Date: Sat, 25 Feb 2017 05:05:55 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: implement utf8 gateway (that
 passes unicode length as a second param)
Message-ID: <58b18133.14542e0a.55522.ce78@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90350:ba2214055259
Date: 2017-02-25 14:05 +0100
http://bitbucket.org/pypy/pypy/changeset/ba2214055259/

Log:	implement utf8 gateway (that passes unicode length as a second
	param)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -264,6 +264,9 @@
     def utf8_w(self, space):
         self._typed_unwrap_error(space, "unicode")
 
+    def convert_to_w_unicode(self, space):
+        self._typed_unwrap_error(space, "unicode")        
+
     def bytearray_list_of_chars_w(self, space):
         self._typed_unwrap_error(space, "bytearray")
 
@@ -1676,8 +1679,12 @@
     def utf8_w(self, w_obj):
         return w_obj.utf8_w(self)
 
+    def convert_to_w_unicode(self, w_obj):
+        return w_obj.convert_to_w_unicode(self)
+
     def unicode0_w(self, w_obj):
         "Like unicode_w, but rejects strings with NUL bytes."
+        xxxx
         from rpython.rlib import rstring
         result = w_obj.unicode_w(self)
         if u'\x00' in result:
@@ -1689,6 +1696,7 @@
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
+        xxx
         if not self.isinstance_w(w_obj, self.w_unicode):
             raise oefmt(self.w_TypeError, "argument must be a unicode")
         return self.unicode_w(w_obj)
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -157,6 +157,9 @@
     def visit_text(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_utf8(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -235,6 +238,7 @@
     def __init__(self):
         UnwrapSpecEmit.__init__(self)
         self.run_args = []
+        self.extracode = []
 
     def scopenext(self):
         return "scope_w[%d]" % self.succ()
@@ -292,6 +296,13 @@
     def visit_text(self, typ):
         self.run_args.append("space.text_w(%s)" % (self.scopenext(),))
 
+    def visit_utf8(self, typ):
+        name = 'w_uni%d' % self.n
+        self.extracode.append('%s = space.convert_to_w_unicode(%s)' %
+                              (name, self.scopenext()))
+        self.run_args.append("space.utf8_w(%s)" % (name,))
+        self.run_args.append("%s._length" % (name,))
+
     def visit_nonnegint(self, typ):
         self.run_args.append("space.gateway_nonnegint_w(%s)" % (
             self.scopenext(),))
@@ -340,8 +351,9 @@
             d = {}
             source = """if 1:
                 def _run(self, space, scope_w):
+                    %s
                     return self.behavior(%s)
-                \n""" % (', '.join(self.run_args),)
+                \n""" % ("\n".join(self.extracode), ', '.join(self.run_args))
             exec compile2(source) in self.miniglobals, d
 
             activation_cls = type("BuiltinActivation_UwS_%s" % label,
@@ -382,6 +394,7 @@
         UnwrapSpecEmit.__init__(self)
         self.args = []
         self.unwrap = []
+        self.extracode = []
         self.finger = 0
 
     def dispatch(self, el, *args):
@@ -448,6 +461,13 @@
     def visit_text(self, typ):
         self.unwrap.append("space.text_w(%s)" % (self.nextarg(),))
 
+    def visit_utf8(self, typ):
+        name = 'w_uni%d' % self.n
+        self.extracode.append('%s = space.convert_to_w_unicode(%s)' %
+                              (name, self.nextarg()))
+        self.unwrap.append("space.utf8_w(%s)" % (name,))
+        self.unwrap.append("%s._length" % (name,))
+
     def visit_nonnegint(self, typ):
         self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),))
 
@@ -472,6 +492,7 @@
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    @staticmethod
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -495,21 +516,21 @@
             unwrap_info.miniglobals['func'] = func
             source = """if 1:
                 def fastfunc_%s_%d(%s):
+                    %s
                     return func(%s)
                 \n""" % (func.__name__.replace('-', '_'), narg,
-                         ', '.join(args),
+                         ', '.join(args), '\n'.join(unwrap_info.extracode),
                          ', '.join(unwrap_info.unwrap))
             exec compile2(source) in unwrap_info.miniglobals, d
             fastfunc = d['fastfunc_%s_%d' % (func.__name__.replace('-', '_'), narg)]
         return narg, fastfunc
-    make_fastfunc = staticmethod(make_fastfunc)
 
 
 def int_unwrapping_space_method(typ):
-    assert typ in (int, str, float, unicode, r_longlong, r_uint, r_ulonglong, bool)
+    assert typ in (int, str, float, r_longlong, r_uint, r_ulonglong, bool)
     if typ is r_int is r_longlong:
         return 'gateway_r_longlong_w'
-    elif typ in (str, unicode, bool):
+    elif typ in (str, bool):
         return typ.__name__ + '_w'
     else:
         return 'gateway_' + typ.__name__ + '_w'
@@ -533,6 +554,13 @@
         return func
     return decorator
 
+def unwrap_count_len(spec):
+    lgt = len(spec)
+    for item in spec:
+        if item == 'utf8':
+            lgt += 1
+    return lgt
+
 class WrappedDefault(object):
     """ Can be used inside unwrap_spec as WrappedDefault(3) which means
     it'll be treated as W_Root, but fed with default which will be a wrapped
@@ -1004,7 +1032,7 @@
             code = self._code
             assert isinstance(code._unwrap_spec, (list, tuple))
             assert isinstance(code._argnames, list)
-            assert len(code._unwrap_spec) == len(code._argnames)
+            assert unwrap_count_len(code._unwrap_spec) == len(code._argnames)
             for i in range(len(code._unwrap_spec)-1, -1, -1):
                 spec = code._unwrap_spec[i]
                 argname = code._argnames[i]
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -535,21 +535,22 @@
         w_app_g3_r = space.wrap(app_g3_r)
         raises(gateway.OperationError,space.call_function,w_app_g3_r,w(1.0))
 
-    def test_interp2app_unwrap_spec_unicode(self):
+    def test_interp2app_unwrap_spec_utf8(self):
         space = self.space
         w = space.wrap
-        def g3_u(space, uni):
-            return space.wrap(len(uni))
+        def g3_u(space, utf8, utf8len):
+            return space.newtuple([space.wrap(len(utf8)), space.wrap(utf8len)])
         app_g3_u = gateway.interp2app_temp(g3_u,
                                          unwrap_spec=[gateway.ObjSpace,
-                                                      unicode])
+                                                      'utf8'])
         w_app_g3_u = space.wrap(app_g3_u)
+        encoded = u"g&#281;&#347;&#263;".encode('utf8')
         assert self.space.eq_w(
-            space.call_function(w_app_g3_u, w(u"foo")),
-            w(3))
+            space.call_function(w_app_g3_u, w(u"g&#281;&#347;&#263;")),
+            space.newtuple([w(len(encoded)), w(4)]))
         assert self.space.eq_w(
-            space.call_function(w_app_g3_u, w("baz")),
-            w(3))
+            space.call_function(w_app_g3_u, w("foo")),
+            space.newtuple([w(3), w(3)]))
         raises(gateway.OperationError, space.call_function, w_app_g3_u,
                w(None))
         raises(gateway.OperationError, space.call_function, w_app_g3_u,
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -373,14 +373,15 @@
 def make_encoder_wrapper(name):
     rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
     assert hasattr(runicode, rname)
-    @unwrap_spec(uni=unicode, errors='str_or_None')
-    def wrap_encoder(space, uni, errors="strict"):
+    @unwrap_spec(uni='utf8', errors='str_or_None')
+    def wrap_encoder(space, utf8, utf8len, errors="strict"):
         if errors is None:
             errors = 'strict'
         state = space.fromcache(CodecState)
         func = getattr(runicode, rname)
-        result = func(uni, len(uni), errors, state.encode_error_handler)
-        return space.newtuple([space.newbytes(result), space.newint(len(uni))])
+        result = func(utf8, len(utf8), utf8len,
+            errors, state.encode_error_handler)
+        return space.newtuple([space.newbytes(result), space.newint(utf8len)])
     wrap_encoder.func_name = rname
     globals()[name] = wrap_encoder
 
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -1,6 +1,6 @@
 """The builtin str implementation"""
 
-from rpython.rlib import jit
+from rpython.rlib import jit, rutf8
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin)
 from rpython.rlib.buffer import StringBuffer
@@ -52,9 +52,17 @@
             uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
         return space.newint(uid)
 
-    def unicode_w(self, space):
+    def convert_to_w_unicode(self, space):
         # Use the default encoding.
         encoding = getdefaultencoding(space)
+        if encoding == 'ascii':
+            try:
+                rutf8.check_ascii(self._value)
+                return space.newutf8(self._value, len(self._value))
+            except rutf8.AsciiCheckError:
+                xxx
+        else:
+            xxx
         return space.unicode_w(decode_object(space, self, encoding, None))
 
     def descr_add(self, space, w_other):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -136,16 +136,19 @@
 
     @staticmethod
     def _op_val(space, w_other, strict=None):
-        return W_UnicodeObject._convert_to_unicode(space, w_other)._utf8.decode('utf8')
+        return W_UnicodeObject.convert_arg_to_w_unicode(space, w_other)._utf8.decode('utf8')
 
     @staticmethod
-    def _convert_to_unicode(space, w_other):
+    def convert_arg_to_w_unicode(space, w_other):
         if isinstance(w_other, W_UnicodeObject):
             return w_other
         if space.isinstance_w(w_other, space.w_bytes):
             return unicode_from_string(space, w_other)
         return unicode_from_encoded_object(space, w_other, None, "strict")
 
+    def convert_to_w_unicode(self, space):
+        return self
+
     def _chr(self, char):
         assert len(char) == 1
         return char[0]
@@ -249,7 +252,7 @@
 
     def descr_eq(self, space, w_other):
         try:
-            res = self._utf8 == self._convert_to_unicode(space, w_other)._utf8
+            res = self._utf8 == self.convert_arg_to_w_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -265,7 +268,7 @@
 
     def descr_ne(self, space, w_other):
         try:
-            res = self._utf8 != self._convert_to_unicode(space, w_other)._utf8
+            res = self._utf8 != self.convert_arg_to_w_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -281,7 +284,7 @@
 
     def descr_lt(self, space, w_other):
         try:
-            res = self._utf8 < self._convert_to_unicode(space, w_other)._utf8
+            res = self._utf8 < self.convert_arg_to_w_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -290,7 +293,7 @@
 
     def descr_le(self, space, w_other):
         try:
-            res = self._utf8 <= self._convert_to_unicode(space, w_other)._utf8
+            res = self._utf8 <= self.convert_arg_to_w_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -299,7 +302,7 @@
 
     def descr_gt(self, space, w_other):
         try:
-            res = self._utf8 > self._convert_to_unicode(space, w_other)._utf8
+            res = self._utf8 > self.convert_arg_to_w_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -308,7 +311,7 @@
 
     def descr_ge(self, space, w_other):
         try:
-            res = self._utf8 >= self._convert_to_unicode(space, w_other)._utf8
+            res = self._utf8 >= self.convert_arg_to_w_unicode(space, w_other)._utf8
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -431,7 +434,7 @@
 
     def descr_add(self, space, w_other):
         try:
-            w_other = self._convert_to_unicode(space, w_other)
+            w_other = self.convert_arg_to_w_unicode(space, w_other)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -458,7 +461,7 @@
                 return self._join_autoconvert(space, list_w)
             # XXX Maybe the extra copy here is okay? It was basically going to
             #     happen anyway, what with being placed into the builder
-            w_u = self._convert_to_unicode(space, w_s)
+            w_u = self.convert_arg_to_w_unicode(space, w_s)
             unwrapped.append(w_u._utf8)
             lgt += w_u._length
             prealloc_size += len(unwrapped[i])
@@ -525,7 +528,7 @@
             res = split(value, maxsplit=maxsplit)
             return space.newlist([W_UnicodeObject(s, -1) for s in res])
 
-        by = self._convert_to_unicode(space, w_sep)._utf8
+        by = self.convert_arg_to_w_unicode(space, w_sep)._utf8
         if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = split(value, by, maxsplit)
@@ -540,7 +543,7 @@
             res = rsplit(value, maxsplit=maxsplit)
             return space.newlist([W_UnicodeObject(s, -1) for s in res])
 
-        by = self._convert_to_unicode(space, w_sep)._utf8
+        by = self.convert_arg_to_w_unicode(space, w_sep)._utf8
         if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = rsplit(value, by, maxsplit)
@@ -550,7 +553,7 @@
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_center(self, space, width, w_fillchar):
         value = self._utf8
-        fillchar = self._convert_to_unicode(space, w_fillchar)._utf8
+        fillchar = self.convert_arg_to_w_unicode(space, w_fillchar)._utf8
         if len(fillchar) != 1:
             raise oefmt(space.w_TypeError,
                         "center() argument 2 must be a single character")

From pypy.commits at gmail.com  Sat Feb 25 09:09:59 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 25 Feb 2017 06:09:59 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: I'll arrive one day earlier
Message-ID: <58b19037.1a0e2e0a.4eca.7c6b@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r5792:4a41227c5ea7
Date: 2017-02-25 15:09 +0100
http://bitbucket.org/pypy/extradoc/changeset/4a41227c5ea7/

Log:	I'll arrive one day earlier

diff --git a/sprintinfo/leysin-winter-2017/people.txt b/sprintinfo/leysin-winter-2017/people.txt
--- a/sprintinfo/leysin-winter-2017/people.txt
+++ b/sprintinfo/leysin-winter-2017/people.txt
@@ -15,7 +15,7 @@
 Manuel Jacob         25.02/04.03    Ermina
 Robert Zaremba       26.02/01.03    Ermina
 Ronan Lamy           25.02/04.03    Ermina
-Antonio Cuni         27.02/03.03    Ermina
+Antonio Cuni         26.02/03.03    Ermina
 Maciej Fijalkowski   26.02/01.03    Ermina
 Stefan Beyer         26.02/04.03    Ermina
 ==================== ============== =======================

From pypy.commits at gmail.com  Sat Feb 25 09:38:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 25 Feb 2017 06:38:12 -0800 (PST)
Subject: [pypy-commit] pypy default: Blind fix: workaround for WSL
Message-ID: <58b196d4.4395190a.2afe7.d265@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90351:6a66210c1f40
Date: 2017-02-25 15:35 +0100
http://bitbucket.org/pypy/pypy/changeset/6a66210c1f40/

Log:	Blind fix: workaround for WSL

diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -197,15 +197,30 @@
     return 0;
 }
 
+static int itimer_which = ITIMER_PROF;
+
 static int install_sigprof_timer(void)
 {
     struct itimerval timer;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = profile_interval_usec;
     timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
+    if (setitimer(itimer_which, &timer, NULL) == 0)
+        return 0;   /* normal path */
+
+    if (errno == EINVAL) {
+        /* on WSL, only ITIMER_REAL is supported */
+        if (setitimer(ITIMER_REAL, &timer, NULL) == 0) {
+            fprintf(stderr, "warning: setitimer(): ITIMER_PROF not "
+                            "available, using ITIMER_REAL instead. "
+                            "Multithreaded programs and programs "
+                            "doing a lot of I/O won't give correct "
+                            "results.\n");
+            itimer_which = ITIMER_REAL;
+            return 0;
+        }
+    }
+    return -1;
 }
 
 static int remove_sigprof_timer(void) {
@@ -214,7 +229,7 @@
     timer.it_interval.tv_usec = 0;
     timer.it_value.tv_sec = 0;
     timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
+    if (setitimer(itimer_which, &timer, NULL) != 0)
         return -1;
     return 0;
 }

From pypy.commits at gmail.com  Sat Feb 25 09:38:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 25 Feb 2017 06:38:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58b196d6.0e572e0a.bbae2.8505@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90352:e0f57c4b95e8
Date: 2017-02-25 15:37 +0100
http://bitbucket.org/pypy/pypy/changeset/e0f57c4b95e8/

Log:	hg merge default

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1245,6 +1245,17 @@
 
 def write_header(header_name, decls):
     lines = [
+        '''
+#ifdef _WIN64
+/* this check is for sanity, but also because the 'temporary fix'
+   below seems to become permanent and would cause unexpected
+   nonsense on Win64---but note that it's not the only reason for
+   why Win64 is not supported!  If you want to help, see
+   http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+   */
+#  error "PyPy does not support 64-bit on Windows.  Use Win32"
+#endif
+''',
         '#define Signed   long           /* xxx temporary fix */',
         '#define Unsigned unsigned long  /* xxx temporary fix */',
         '',] + decls + [
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -73,7 +73,7 @@
 
 #define Py_MEMCPY memcpy
 
-#include <pypy_macros.h>
+#include "pypy_macros.h"
 
 #define PyExc_EnvironmentError PyExc_OSError
 #define PyExc_IOError PyExc_OSError
@@ -144,7 +144,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-  #include <pypy_decl.h>
+  #include "pypy_decl.h"
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h b/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
--- a/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
+++ b/pypy/module/cpyext/include/_numpypy/numpy/ndarraytypes.h
@@ -1,7 +1,7 @@
 #ifndef NDARRAYTYPES_H
 #define NDARRAYTYPES_H
 
-#include "numpy/npy_common.h"
+#include "npy_common.h"
 //#include "npy_endian.h"
 //#include "npy_cpu.h"
 //#include "utils.h"
diff --git a/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
--- a/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
+++ b/pypy/module/cpyext/include/_numpypy/numpy/npy_3kcompat.h
@@ -10,7 +10,7 @@
 #ifndef _NPY_3KCOMPAT_H_
 #define _NPY_3KCOMPAT_H_
 
-#include <numpy/npy_common.h>
+#include "npy_common.h"
 
 #define npy_PyFile_Dup(file, mode) (NULL)
 #define npy_PyFile_DupClose(file, handle) (0)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-#include <cpyext_object.h>
+#include "cpyext_object.h"
 
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -5,7 +5,7 @@
 extern "C" {
 #endif
 
-#include <cpyext_unicodeobject.h>
+#include "cpyext_unicodeobject.h"
 
 /* Fast access macros */
 #ifndef Py_LIMITED_API
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -289,7 +289,7 @@
     destructor tp_finalize;
 } PyTypeObject;
 
-typedef struct {
+typedef struct _heaptypeobject {
     PyTypeObject ht_type;
     PyNumberMethods as_number;
     PyMappingMethods as_mapping;
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4840,6 +4840,21 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_nonvirtual_newstr_strlen(self):
+        ops = """
+        [p0]
+        p1 = call_r(0, p0, s"X", descr=strconcatdescr)
+        i0 = strlen(p1)
+        finish(i0)
+        """
+        expected = """
+        [p0]
+        i2 = strlen(p0)
+        i4 = int_add(i2, 1)
+        finish(i4)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
     def test_copy_long_string_to_virtual(self):
         ops = """
         []
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -60,7 +60,6 @@
         self.length = length
         self._is_virtual = is_virtual
         self.mode = mode
-        self.length = length
 
     def getlenbound(self, mode):
         from rpython.jit.metainterp.optimizeopt import intutils
@@ -110,12 +109,11 @@
         return self.string_copy_parts(op, string_optimizer, targetbox,
                                       offsetbox, mode)
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         if self.lgtop is not None:
             return self.lgtop
         assert not self.is_virtual()
-        if not create_ops:
-            return None
         lengthop = ResOperation(mode.STRLEN, [op])
         lengthop.set_forwarded(self.getlenbound(mode))
         self.lgtop = lengthop
@@ -172,7 +170,8 @@
     def is_virtual(self):
         return self._is_virtual
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         if self.lgtop is None:
             self.lgtop = ConstInt(len(self._chars))
         return self.lgtop
@@ -251,7 +250,8 @@
             return s1[start : start + length]
         return None
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         return self.lgtop
 
     def _visitor_walk_recursive(self, instbox, visitor, optimizer):
@@ -280,20 +280,19 @@
     def is_virtual(self):
         return self._is_virtual
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
+        assert op is not None
         if self.lgtop is not None:
             return self.lgtop
         lefti = string_optimizer.getptrinfo(self.vleft)
-        len1box = lefti.getstrlen(self.vleft, string_optimizer, mode,
-                                  create_ops)
+        len1box = lefti.getstrlen(self.vleft, string_optimizer, mode)
         if len1box is None:
             return None
         righti = string_optimizer.getptrinfo(self.vright)
-        len2box = righti.getstrlen(self.vright, string_optimizer, mode,
-                                   create_ops)
+        len2box = righti.getstrlen(self.vright, string_optimizer, mode)
         if len2box is None:
             return None
-        self.lgtop = _int_add(string_optimizer, len1box, len2box, create_ops)
+        self.lgtop = _int_add(string_optimizer, len1box, len2box)
             # ^^^ may still be None, if string_optimizer is None
         return self.lgtop
 
@@ -371,7 +370,7 @@
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(string_optimizer, box1, box2, create_ops=True):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -379,8 +378,6 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if not create_ops:
-        return None
     op = ResOperation(rop.INT_ADD, [box1, box2])
     string_optimizer.send_extra_operation(op)
     return op
@@ -529,9 +526,10 @@
         return self._optimize_STRLEN(op, mode_unicode)
 
     def _optimize_STRLEN(self, op, mode):
-        opinfo = self.getptrinfo(op.getarg(0))
+        arg1 = self.get_box_replacement(op.getarg(0))
+        opinfo = self.getptrinfo(arg1)
         if opinfo:
-            lgtop = opinfo.getstrlen(op, self, mode, False)
+            lgtop = opinfo.getstrlen(arg1, self, mode)
             if lgtop is not None:
                 self.make_equal_to(op, lgtop)
                 return
@@ -700,11 +698,11 @@
         i2 = self.getptrinfo(arg2)
         #
         if i1:
-            l1box = i1.getstrlen(arg1, self, mode, create_ops=False)
+            l1box = i1.getstrlen(arg1, self, mode)
         else:
             l1box = None
         if i2:
-            l2box = i2.getstrlen(arg2, self, mode, create_ops=False)
+            l2box = i2.getstrlen(arg2, self, mode)
         else:
             l2box = None
         if (l1box is not None and l2box is not None and
@@ -742,7 +740,7 @@
         l2box = None
         l1box = None
         if i2:
-            l2box = i2.getstrlen(arg2, self, mode, create_ops=False)
+            l2box = i2.getstrlen(arg2, self, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
                 if i1 and i1.is_nonnull():
@@ -760,7 +758,7 @@
                     return True, None
             if l2box.value == 1:
                 if i1:
-                    l1box = i1.getstrlen(arg1, self, mode, False)
+                    l1box = i1.getstrlen(arg1, self, mode)
                 if isinstance(l1box, ConstInt) and l1box.value == 1:
                     # comparing two single chars
                     vchar1 = self.strgetitem(None, arg1, optimizer.CONST_0,
@@ -799,7 +797,7 @@
         i2 = self.getptrinfo(arg2)
         l2box = None
         if i2:
-            l2box = i2.getstrlen(arg1, self, mode, create_ops=False)
+            l2box = i2.getstrlen(arg1, self, mode)
         if l2box:
             l2info = self.getintbound(l2box)
             if l2info.is_constant():
@@ -822,12 +820,14 @@
         return False, None
 
     def opt_call_stroruni_STR_CMP(self, op, mode):
-        i1 = self.getptrinfo(op.getarg(1))
-        i2 = self.getptrinfo(op.getarg(2))
+        arg1 = self.get_box_replacement(op.getarg(1))
+        arg2 = self.get_box_replacement(op.getarg(2))
+        i1 = self.getptrinfo(arg1)
+        i2 = self.getptrinfo(arg2)
         if not i1 or not i2:
             return False, None
-        l1box = i1.getstrlen(None, self, mode, False)
-        l2box = i2.getstrlen(None, self, mode, False)
+        l1box = i1.getstrlen(arg1, self, mode)
+        l2box = i2.getstrlen(arg2, self, mode)
         if (l1box is not None and l2box is not None and
             isinstance(l1box, ConstInt) and
             isinstance(l2box, ConstInt) and
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -197,15 +197,30 @@
     return 0;
 }
 
+static int itimer_which = ITIMER_PROF;
+
 static int install_sigprof_timer(void)
 {
     struct itimerval timer;
     timer.it_interval.tv_sec = 0;
     timer.it_interval.tv_usec = profile_interval_usec;
     timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
+    if (setitimer(itimer_which, &timer, NULL) == 0)
+        return 0;   /* normal path */
+
+    if (errno == EINVAL) {
+        /* on WSL, only ITIMER_REAL is supported */
+        if (setitimer(ITIMER_REAL, &timer, NULL) == 0) {
+            fprintf(stderr, "warning: setitimer(): ITIMER_PROF not "
+                            "available, using ITIMER_REAL instead. "
+                            "Multithreaded programs and programs "
+                            "doing a lot of I/O won't give correct "
+                            "results.\n");
+            itimer_which = ITIMER_REAL;
+            return 0;
+        }
+    }
+    return -1;
 }
 
 static int remove_sigprof_timer(void) {
@@ -214,7 +229,7 @@
     timer.it_interval.tv_usec = 0;
     timer.it_value.tv_sec = 0;
     timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
+    if (setitimer(itimer_which, &timer, NULL) != 0)
         return -1;
     return 0;
 }
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1730,7 +1730,11 @@
                      (ulimitv, ' '.join(args),)]
             popen = subprocess.Popen(args1, stderr=subprocess.PIPE)
             _, child_stderr = popen.communicate()
-            assert popen.wait() == 134     # aborted
+            assert popen.wait() in (-6, 134)     # aborted
+            # note: it seems that on some systems we get 134 and on
+            # others we get -6.  Bash is supposed to translate the
+            # SIGABRT (signal 6) from the subprocess into the exit 
+            # code 128+6, but I guess it may not always do so.
             assert 'out of memory:' in child_stderr
             return '42'
         #

From pypy.commits at gmail.com  Sat Feb 25 09:48:37 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 25 Feb 2017 06:48:37 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add passing test
Message-ID: <58b19945.8f1f190a.ac3df.d335@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90353:855aac3ae3fc
Date: 2017-02-25 14:48 +0000
http://bitbucket.org/pypy/pypy/changeset/855aac3ae3fc/

Log:	Add passing test

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -229,6 +229,13 @@
         assert d[cmpr] == 72
         assert d[3] == 72
 
+    def test_hash_inheritance(self):
+        foo = self.import_module("foo")
+        assert hash(foo.UnicodeSubtype(u'xyz')) == hash(u'xyz')
+        assert foo.UnicodeSubtype.__hash__ is str.__hash__
+        assert hash(foo.UnicodeSubtype3(u'xyz')) == hash(u'xyz')
+        assert foo.UnicodeSubtype3.__hash__ is str.__hash__
+
     def test_descriptor(self):
         module = self.import_module("foo")
         prop = module.Property()
@@ -1139,8 +1146,8 @@
                   ((PyHeapTypeObject*)Base2)->ht_qualname = dummyname;
                   ((PyHeapTypeObject*)Base12)->ht_qualname = dummyname;
                 }
-                #endif 
-                #endif 
+                #endif
+                #endif
                 Base1->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
                 Base2->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
                 Base12->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE;

From pypy.commits at gmail.com  Sat Feb 25 10:18:43 2017
From: pypy.commits at gmail.com (fijal)
Date: Sat, 25 Feb 2017 07:18:43 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: get enough plumbing to start
 running more tests
Message-ID: <58b1a053.11162e0a.87141.77a2@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90354:6af99b57c74f
Date: 2017-02-25 16:18 +0100
http://bitbucket.org/pypy/pypy/changeset/6af99b57c74f/

Log:	get enough plumbing to start running more tests

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -159,6 +159,7 @@
 
     def visit_utf8(self, el, app_sig):
         self.checked_space_method(el, app_sig)
+        self.orig_arg() # iterate
 
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
@@ -607,6 +608,16 @@
                              "the name of an argument of the following "
                              "function" % (name,))
 
+    if kw_spec:
+        filtered = []
+        i = 0
+        while i < len(unwrap_spec):
+            elem = unwrap_spec[i]
+            filtered.append(elem)
+            if elem == 'utf8':
+                i += 1
+            i += 1
+        unwrap_spec = filtered
     return unwrap_spec
 
 
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -74,3 +74,8 @@
         uni, len(uni), "strict",
         errorhandler=raise_unicode_exception_encode,
         allow_surrogates=True)
+
+def utf8_encode_ascii(utf8, utf8len, errors, errorhandler):
+    if len(utf8) == utf8len:
+        return utf8
+    xxx
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -371,15 +371,16 @@
 from rpython.rlib import runicode
 
 def make_encoder_wrapper(name):
-    rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
-    assert hasattr(runicode, rname)
-    @unwrap_spec(uni='utf8', errors='str_or_None')
+    rname = "utf8_encode_%s" % (name.replace("_encode", ""), )
+    @unwrap_spec(utf8='utf8', errors='str_or_None')
     def wrap_encoder(space, utf8, utf8len, errors="strict"):
+        from pypy.interpreter import unicodehelper
+
         if errors is None:
             errors = 'strict'
         state = space.fromcache(CodecState)
-        func = getattr(runicode, rname)
-        result = func(utf8, len(utf8), utf8len,
+        func = getattr(unicodehelper, rname)
+        result = func(utf8, utf8len,
             errors, state.encode_error_handler)
         return space.newtuple([space.newbytes(result), space.newint(utf8len)])
     wrap_encoder.func_name = rname
@@ -438,10 +439,11 @@
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=True"
- at unwrap_spec(uni=unicode, errors='str_or_None')
-def utf_8_encode(space, uni, errors="strict"):
+ at unwrap_spec(utf8='utf8', errors='str_or_None')
+def utf_8_encode(space, utf8, utf8len, errors="strict"):
     if errors is None:
         errors = 'strict'
+    xxx
     state = space.fromcache(CodecState)
     # NB. can't call unicode_encode_utf_8() directly because that's
     # an @elidable function nowadays.  Instead, we need the _impl().
@@ -605,8 +607,9 @@
         final, state.decode_error_handler, mapping)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
- at unwrap_spec(uni=unicode, errors='str_or_None')
-def charmap_encode(space, uni, errors="strict", w_mapping=None):
+ at unwrap_spec(utf8='utf8', errors='str_or_None')
+def charmap_encode(space, utf8, utf8len, errors="strict", w_mapping=None):
+    xxx
     if errors is None:
         errors = 'strict'
     if space.is_none(w_mapping):
@@ -621,9 +624,10 @@
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
 
- at unwrap_spec(chars=unicode)
-def charmap_build(space, chars):
+ at unwrap_spec(chars='utf8')
+def charmap_build(space, chars, charslen):
     # XXX CPython sometimes uses a three-level trie
+    xxx
     w_charmap = space.newdict()
     for num in range(len(chars)):
         elem = chars[num]
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -4,7 +4,7 @@
     compute_hash, compute_unique_id, import_from_mixin,
     enforceargs, newlist_hint)
 from rpython.rlib.buffer import StringBuffer
-from rpython.rlib.rstring import StringBuilder, split, rsplit
+from rpython.rlib.rstring import StringBuilder, split, rsplit, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
@@ -111,7 +111,7 @@
         return space.newint(ord(self._value[0]))
 
     def _new(self, value):
-        return W_UnicodeObject(value.encode('utf8', len(value)))
+        return W_UnicodeObject(value.encode('utf8'), len(value))
 
     def _new_from_list(self, value):
         xxx
@@ -153,7 +153,7 @@
         assert len(char) == 1
         return char[0]
 
-    _builder = StringBuilder
+    _builder = UnicodeBuilder
 
     def _isupper(self, ch):
         return unicodedb.isupper(ord(ch))
@@ -411,7 +411,8 @@
 
     def descr_islower(self, space):
         cased = False
-        for uchar in self._value:
+        val = self._val(space)
+        for uchar in val:
             if (unicodedb.isupper(ord(uchar)) or
                 unicodedb.istitle(ord(uchar))):
                 return space.w_False
@@ -421,7 +422,7 @@
 
     def descr_isupper(self, space):
         cased = False
-        for uchar in self._value:
+        for uchar in self._val(space):
             if (unicodedb.islower(ord(uchar)) or
                 unicodedb.istitle(ord(uchar))):
                 return space.w_False
@@ -466,7 +467,7 @@
             lgt += w_u._length
             prealloc_size += len(unwrapped[i])
 
-        sb = self._builder(prealloc_size)
+        sb = StringBuilder(prealloc_size)
         for i in range(size):
             if value and i != 0:
                 sb.append(value)
@@ -508,7 +509,7 @@
         if num_zeros <= 0:
             # cannot return self, in case it is a subclass of str
             return W_UnicodeObject(selfval, self._len())
-        builder = self._builder(num_zeros + len(selfval))
+        builder = StringBuilder(num_zeros + len(selfval))
         if len(selfval) > 0 and (selfval[0] == '+' or selfval[0] == '-'):
             # copy sign to first position
             builder.append(selfval[0])
@@ -569,9 +570,6 @@
 
         return W_UnicodeObject(centered, self._len() + d)
 
-    def descr_title(self, space):
-        return 
-
 def wrapunicode(space, uni):
     return W_UnicodeObject(uni)
 

From pypy.commits at gmail.com  Sat Feb 25 10:33:16 2017
From: pypy.commits at gmail.com (fijal)
Date: Sat, 25 Feb 2017 07:33:16 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: enough plumbing to pass some tests,
 not efficient at all
Message-ID: <58b1a3bc.13542e0a.d079d.ab78@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90356:522a73b2ba2e
Date: 2017-02-25 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/522a73b2ba2e/

Log:	enough plumbing to pass some tests, not efficient at all

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -702,7 +702,7 @@
         if space.isinstance_w(w_prefix, space.w_unicode):
             self_as_unicode = unicode_from_encoded_object(space, self, None,
                                                           None)
-            return self_as_unicode._startswith(space, self_as_unicode._value,
+            return self_as_unicode._startswith(space, self_as_unicode._utf8.decode("utf8"),
                                                w_prefix, start, end)
         return self._StringMethods__startswith(space, value, w_prefix, start,
                                                end)
@@ -712,7 +712,7 @@
         if space.isinstance_w(w_suffix, space.w_unicode):
             self_as_unicode = unicode_from_encoded_object(space, self, None,
                                                           None)
-            return self_as_unicode._endswith(space, self_as_unicode._value,
+            return self_as_unicode._endswith(space, self_as_unicode._utf8.decode("utf8"),
                                              w_suffix, start, end)
         return self._StringMethods__endswith(space, value, w_suffix, start,
                                              end)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -237,7 +237,7 @@
 
         assert isinstance(w_value, W_UnicodeObject)
         w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype)
-        W_UnicodeObject.__init__(w_newobj, w_value._value)
+        W_UnicodeObject.__init__(w_newobj, w_value._utf8, w_value._length)
         return w_newobj
 
     def descr_repr(self, space):
@@ -340,8 +340,7 @@
         return mod_format(space, w_values, self, do_unicode=True)
 
     def descr_translate(self, space, w_table):
-        xxx
-        selfvalue = self._value
+        selfvalue = self._utf8.decode("utf8")
         w_sys = space.getbuiltinmodule('sys')
         maxunicode = space.int_w(space.getattr(w_sys,
                                                space.newtext("maxunicode")))
@@ -365,12 +364,12 @@
                                     hex(maxunicode + 1))
                     result.append(unichr(newval))
                 elif space.isinstance_w(w_newval, space.w_unicode):
-                    result.append(space.unicode_w(w_newval))
+                    result.append(space.utf8_w(w_newval).decode("utf8"))
                 else:
                     raise oefmt(space.w_TypeError,
                                 "character mapping must return integer, None "
                                 "or unicode")
-        return W_UnicodeObject(u''.join(result))
+        return W_UnicodeObject(u''.join(result).encode("utf8"), -1)
 
     def descr_encode(self, space, w_encoding=None, w_errors=None):
         encoding, errors = _get_encoding_and_errors(space, w_encoding,
@@ -1286,7 +1285,8 @@
 def unicode_to_decimal_w(space, w_unistr):
     if not isinstance(w_unistr, W_UnicodeObject):
         raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr)
-    unistr = w_unistr._value
+    unistr = w_unistr._utf8.decode("utf8")
+    # XXX speed up
     result = ['\0'] * len(unistr)
     digits = ['0', '1', '2', '3', '4',
               '5', '6', '7', '8', '9']

From pypy.commits at gmail.com  Sat Feb 25 10:33:14 2017
From: pypy.commits at gmail.com (fijal)
Date: Sat, 25 Feb 2017 07:33:14 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: fix
Message-ID: <58b1a3ba.0d57190a.e5f64.d370@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90355:646678d11ad8
Date: 2017-02-25 16:27 +0100
http://bitbucket.org/pypy/pypy/changeset/646678d11ad8/

Log:	fix

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -136,14 +136,17 @@
 
     @staticmethod
     def _op_val(space, w_other, strict=None):
-        return W_UnicodeObject.convert_arg_to_w_unicode(space, w_other)._utf8.decode('utf8')
+        return W_UnicodeObject.convert_arg_to_w_unicode(space, w_other, strict)._utf8.decode('utf8')
 
     @staticmethod
-    def convert_arg_to_w_unicode(space, w_other):
+    def convert_arg_to_w_unicode(space, w_other, strict=None):
         if isinstance(w_other, W_UnicodeObject):
             return w_other
         if space.isinstance_w(w_other, space.w_bytes):
             return unicode_from_string(space, w_other)
+        if strict:
+            raise oefmt(space.w_TypeError,
+                "%s arg must be None, unicode or str", strict)
         return unicode_from_encoded_object(space, w_other, None, "strict")
 
     def convert_to_w_unicode(self, space):

From pypy.commits at gmail.com  Sat Feb 25 15:54:36 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 25 Feb 2017 12:54:36 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges-3: merge default
Message-ID: <58b1ef0c.85d3190a.72197.e53f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges-3
Changeset: r90357:8c7255d2aefd
Date: 2017-02-24 13:02 +0100
http://bitbucket.org/pypy/pypy/changeset/8c7255d2aefd/

Log:	merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -74,7 +74,7 @@
 ^rpython/doc/_build/.*$
 ^compiled
 ^.git/
-^.hypothesis/
+.hypothesis/
 ^release/
 ^rpython/_cache$
 
diff --git a/extra_tests/README.txt b/extra_tests/README.txt
new file mode 100644
--- /dev/null
+++ b/extra_tests/README.txt
@@ -0,0 +1,5 @@
+The tests in this directory are a complement to lib-python/3/test/.
+
+They are meant to run on top of a compiled pypy3 or CPython3.5 in an
+environment containing at least pytest and hypothesis, using a command like
+'pytest extra_tests/'.
diff --git a/extra_tests/pytest.ini b/extra_tests/pytest.ini
new file mode 100644
diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_unicode.py
@@ -0,0 +1,34 @@
+import pytest
+from hypothesis import strategies as st
+from hypothesis import given, settings, example
+
+from unicodedata import normalize
+
+# For every (n1, n2, n3) triple, applying n1 then n2 must be the same
+# as applying n3.
+# Reference: http://unicode.org/reports/tr15/#Design_Goals
+compositions = [
+    ('NFC', 'NFC', 'NFC'),
+    ('NFC', 'NFD', 'NFD'),
+    ('NFC', 'NFKC', 'NFKC'),
+    ('NFC', 'NFKD', 'NFKD'),
+    ('NFD', 'NFC', 'NFC'),
+    ('NFD', 'NFD', 'NFD'),
+    ('NFD', 'NFKC', 'NFKC'),
+    ('NFD', 'NFKD', 'NFKD'),
+    ('NFKC', 'NFC', 'NFKC'),
+    ('NFKC', 'NFD', 'NFKD'),
+    ('NFKC', 'NFKC', 'NFKC'),
+    ('NFKC', 'NFKD', 'NFKD'),
+    ('NFKD', 'NFC', 'NFKC'),
+    ('NFKD', 'NFD', 'NFKD'),
+    ('NFKD', 'NFKC', 'NFKC'),
+    ('NFKD', 'NFKD', 'NFKD'),
+]
+
+ at pytest.mark.parametrize('norm1, norm2, norm3', compositions)
+ at settings(max_examples=1000)
+ at example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at given(s=st.text())
+def test_composition(s, norm1, norm2, norm3):
+    assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -158,6 +158,8 @@
 not on PyPy 3.x.  The latter is used to get an app-level unicode string
 by decoding the RPython string, assumed to be utf-8.
 
+.. branch: space-wrap
+
 .. branch: fix_bool_restype
 
 Fix for ``ctypes.c_bool``-returning ctypes functions
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -178,7 +178,6 @@
             ("bounce", "METH_NOARGS",
             """
             PyGILState_STATE gilstate;
-            PyThreadState *tstate;
             PyObject *dict;
 
             if (PyEval_ThreadsInitialized() == 0)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -5,7 +5,7 @@
 from rpython.rlib.unicodedata import unicodedb
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import jit
+from rpython.rlib import jit, nonconst
 
 
 if rffi.sizeof(lltype.UniChar) == 4:
@@ -373,7 +373,12 @@
                             pos += 1
                             _encodeUCS4(result, ch3)
                             continue
-                    if not allow_surrogates:
+                    # note: if the program only ever calls this with
+                    # allow_surrogates=True, then we'll never annotate
+                    # the following block of code, and errorhandler()
+                    # will never be called.  This causes RPython
+                    # problems.  Avoid it with the nonconst hack.
+                    if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
                                                    s, pos-1, pos)
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1730,7 +1730,11 @@
                      (ulimitv, ' '.join(args),)]
             popen = subprocess.Popen(args1, stderr=subprocess.PIPE)
             _, child_stderr = popen.communicate()
-            assert popen.wait() == 134     # aborted
+            assert popen.wait() in (-6, 134)     # aborted
+            # note: it seems that on some systems we get 134 and on
+            # others we get -6.  Bash is supposed to translate the
+            # SIGABRT (signal 6) from the subprocess into the exit 
+            # code 128+6, but I guess it may not always do so.
             assert 'out of memory:' in child_stderr
             return '42'
         #

From pypy.commits at gmail.com  Sat Feb 25 15:54:38 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 25 Feb 2017 12:54:38 -0800 (PST)
Subject: [pypy-commit] pypy default: merge optinfo-into-bridges-3
Message-ID: <58b1ef0e.5125190a.807a2.55f7@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r90358:e2a82bc8b3d4
Date: 2017-02-25 21:53 +0100
http://bitbucket.org/pypy/pypy/changeset/e2a82bc8b3d4/

Log:	merge optinfo-into-bridges-3

	Improve the optimization of branchy Python code by retaining more
	information across failing guards. This is done by appending some
	carefully encoded extra information into the resume code. So far the
	following information is encoded:

	- which boxes have known classes
	- known fields of instances

	The branch also does some cleanup in resume.py.

	Thanks fijal for the review.

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -143,6 +143,13 @@
 behavior is also available on Python 2.x or for the ``dict`` type by
 calling ``__pypy__.move_to_end(dict, key, last=True)``.
 
+
+.. branch optinfo-into-bridges-3
+
+Improve the optimization of branchy Python code by retaining more information
+across failing guards.
+
+
 .. branch: space-newtext
 
 Internal refactoring of ``space.wrap()``, which is now replaced with
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -146,14 +146,13 @@
     def get_register_index_f(self, index):
         return ord(self.live_f[index])
 
-    def enumerate_vars(self, callback_i, callback_r, callback_f, spec, index):
+    def enumerate_vars(self, callback_i, callback_r, callback_f, spec):
         for i in range(self.get_register_count_i()):
-            index = callback_i(index, self.get_register_index_i(i))
+            callback_i(self.get_register_index_i(i))
         for i in range(self.get_register_count_r()):
-            index = callback_r(index, self.get_register_index_r(i))
+            callback_r(self.get_register_index_r(i))
         for i in range(self.get_register_count_f()):
-            index = callback_f(index, self.get_register_index_f(i))
-        return index
+            callback_f(self.get_register_index_f(i))
     enumerate_vars._annspecialcase_ = 'specialize:arg(4)'
 
 _liveness_cache = {}
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -85,13 +85,14 @@
     """ This represents ops() with a jump at the end that goes to some
     loop, we need to deal with virtual state and inlining of short preamble
     """
-    def __init__(self, trace, runtime_boxes, call_pure_results=None,
+    def __init__(self, trace, runtime_boxes, resumestorage=None, call_pure_results=None,
                  enable_opts=None, inline_short_preamble=False):
         self.trace = trace
         self.runtime_boxes = runtime_boxes
         self.call_pure_results = call_pure_results
         self.enable_opts = enable_opts
         self.inline_short_preamble = inline_short_preamble
+        self.resumestorage = resumestorage
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -100,7 +101,8 @@
         return opt.optimize_bridge(self.trace, self.runtime_boxes,
                                    self.call_pure_results,
                                    self.inline_short_preamble,
-                                   self.box_names_memo)
+                                   self.box_names_memo,
+                                   self.resumestorage)
 
 class UnrolledLoopData(CompileData):
     """ This represents label() ops jump with extra info that's from the
@@ -870,10 +872,9 @@
 
 
 class ResumeGuardDescr(AbstractResumeGuardDescr):
-    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+    _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals',
                'rd_pendingfields', 'status')
     rd_numb = lltype.nullptr(NUMBERING)
-    rd_count = 0
     rd_consts = None
     rd_virtuals = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
@@ -882,7 +883,6 @@
         if isinstance(other, ResumeGuardCopiedDescr):
             other = other.prev
         assert isinstance(other, ResumeGuardDescr)
-        self.rd_count = other.rd_count
         self.rd_consts = other.rd_consts
         self.rd_pendingfields = other.rd_pendingfields
         self.rd_virtuals = other.rd_virtuals
@@ -895,7 +895,6 @@
 
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
-        self.rd_count = len(boxes)
         self.store_hash(metainterp_sd)
 
     def clone(self):
@@ -1077,7 +1076,15 @@
     call_pure_results = metainterp.call_pure_results
 
     if metainterp.history.ends_with_jump:
-        data = BridgeCompileData(trace, runtime_boxes,
+        if isinstance(resumekey, ResumeGuardCopiedDescr):
+            key = resumekey.prev
+            assert isinstance(key, ResumeGuardDescr)
+        elif isinstance(resumekey, ResumeFromInterpDescr):
+            key = None
+        else:
+            key = resumekey
+            assert isinstance(key, ResumeGuardDescr)
+        data = BridgeCompileData(trace, runtime_boxes, key,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts,
                                  inline_short_preamble=inline_short_preamble)
diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py
--- a/rpython/jit/metainterp/opencoder.py
+++ b/rpython/jit/metainterp/opencoder.py
@@ -66,7 +66,7 @@
         assert isinstance(snapshot, TopSnapshot)
         self.vable_array = snapshot.vable_array
         self.vref_array = snapshot.vref_array
-        self.size = len(self.vable_array) + len(self.vref_array) + 2
+        self.size = len(self.vable_array) + len(self.vref_array) + 3
         jc_index, pc = unpack_uint(snapshot.packed_jitcode_pc)
         self.framestack = []
         if jc_index == 2**16-1:
diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -0,0 +1,135 @@
+""" Code to feed information from the optimizer via the resume code into the
+optimizer of the bridge attached to a guard. """
+
+from rpython.jit.metainterp import resumecode
+
+
+# adds the following sections at the end of the resume code:
+#
+# ---- known classes
+# <bitfield> size is the number of reference boxes in the liveboxes
+#            1 klass known
+#            0 klass unknown
+#            (the class is found by actually looking at the runtime value)
+#            the bits are bunched in bunches of 7
+#
+# ---- heap knowledge
+# <length>
+# (<box1> <descr> <box2>) length times, if getfield(box1, descr) == box2
+#                         both boxes should be in the liveboxes
+#
+# ----
+
+
+# maybe should be delegated to the optimization classes?
+
+def tag_box(box, liveboxes_from_env, memo):
+    from rpython.jit.metainterp.history import Const
+    if isinstance(box, Const):
+        return memo.getconst(box)
+    else:
+        return liveboxes_from_env[box] # has to exist
+
+def decode_box(resumestorage, tagged, liveboxes, cpu):
+    from rpython.jit.metainterp.resume import untag, TAGCONST, TAGINT, TAGBOX
+    from rpython.jit.metainterp.resume import NULLREF, TAG_CONST_OFFSET, tagged_eq
+    from rpython.jit.metainterp.history import ConstInt
+    num, tag = untag(tagged)
+    # NB: the TAGVIRTUAL case can't happen here, because this code runs after
+    # virtuals are already forced again
+    if tag == TAGCONST:
+        if tagged_eq(tagged, NULLREF):
+            box = cpu.ts.CONST_NULL
+        else:
+            box = resumestorage.rd_consts[num - TAG_CONST_OFFSET]
+    elif tag == TAGINT:
+        box = ConstInt(num)
+    elif tag == TAGBOX:
+        box = liveboxes[num]
+    else:
+        raise AssertionError("unreachable")
+    return box
+
+def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo):
+    available_boxes = {}
+    for box in liveboxes:
+        if box is not None and box in liveboxes_from_env:
+            available_boxes[box] = None
+    metainterp_sd = optimizer.metainterp_sd
+
+    # class knowledge is stored as bits, true meaning the class is known, false
+    # means unknown. on deserializing we look at the bits, and read the runtime
+    # class for the known classes (which has to be the same in the bridge) and
+    # mark that as known. this works for guard_class too: the class is only
+    # known *after* the guard
+    bitfield = 0
+    shifts = 0
+    for box in liveboxes:
+        if box is None or box.type != "r":
+            continue
+        info = optimizer.getptrinfo(box)
+        known_class = info is not None and info.get_known_class(optimizer.cpu) is not None
+        bitfield <<= 1
+        bitfield |= known_class
+        shifts += 1
+        if shifts == 6:
+            numb_state.append_int(bitfield)
+            bitfield = shifts = 0
+    if shifts:
+        numb_state.append_int(bitfield << (6 - shifts))
+
+    # heap knowledge: we store triples of known heap fields in non-virtual
+    # structs
+    # XXX could be extended to arrays
+    if optimizer.optheap:
+        triples = optimizer.optheap.serialize_optheap(available_boxes)
+        # can only encode descrs that have a known index into
+        # metainterp_sd.all_descrs
+        triples = [triple for triple in triples if triple[1].descr_index != -1]
+        numb_state.append_int(len(triples))
+        for box1, descr, box2 in triples:
+            index = descr.descr_index
+            numb_state.append_short(tag_box(box1, liveboxes_from_env, memo))
+            numb_state.append_int(index)
+            numb_state.append_short(tag_box(box2, liveboxes_from_env, memo))
+    else:
+        numb_state.append_int(0)
+
+def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes):
+    reader = resumecode.Reader(resumestorage.rd_numb)
+    assert len(frontend_boxes) == len(liveboxes)
+    metainterp_sd = optimizer.metainterp_sd
+
+    # skip resume section
+    startcount = reader.next_item()
+    reader.jump(startcount - 1)
+
+    # class knowledge
+    bitfield = 0
+    mask = 0
+    for i, box in enumerate(liveboxes):
+        if box.type != "r":
+            continue
+        if not mask:
+            bitfield = reader.next_item()
+            mask = 0b100000
+        class_known = bitfield & mask
+        mask >>= 1
+        if class_known:
+            cls = optimizer.cpu.ts.cls_of_box(frontend_boxes[i])
+            optimizer.make_constant_class(box, cls)
+
+    # heap knowledge
+    if not optimizer.optheap:
+        return
+    length = reader.next_item()
+    result = []
+    for i in range(length):
+        tagged = reader.next_item()
+        box1 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
+        index = reader.next_item()
+        descr = metainterp_sd.all_descrs[index]
+        tagged = reader.next_item()
+        box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
+        result.append((box1, descr, box2))
+    optimizer.optheap.deserialize_optheap(result)
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -48,6 +48,7 @@
         # that has a non-None entry at
         # info._fields[descr.get_index()]
         # must be in cache_infos
+        assert structop.type == 'r'
         self.cached_structs.append(structop)
         self.cached_infos.append(info)
 
@@ -693,6 +694,36 @@
         self._seen_guard_not_invalidated = True
         return self.emit(op)
 
+    def serialize_optheap(self, available_boxes):
+        result = []
+        for descr, cf in self.cached_fields.iteritems():
+            if cf._lazy_set:
+                continue # XXX safe default for now
+            parent_descr = descr.get_parent_descr()
+            if not parent_descr.is_object():
+                continue # XXX could be extended to non-instance objects
+            for i, box1 in enumerate(cf.cached_structs):
+                if box1 not in available_boxes:
+                    continue
+                structinfo = cf.cached_infos[i]
+                box2 = structinfo.getfield(descr).get_box_replacement()
+                if isinstance(box2, Const) or box2 in available_boxes:
+                    result.append((box1, descr, box2))
+        return result
+
+    def deserialize_optheap(self, triples):
+        for box1, descr, box2 in triples:
+            parent_descr = descr.get_parent_descr()
+            assert parent_descr.is_object()
+            structinfo = box1.get_forwarded()
+            if not isinstance(structinfo, info.AbstractVirtualPtrInfo):
+                structinfo = info.InstancePtrInfo(parent_descr)
+                structinfo.init_fields(parent_descr, descr.get_index())
+                box1.set_forwarded(structinfo)
+
+            cf = self.field_cache(descr)
+            structinfo.setfield(descr, box1, box2, optheap=self, cf=cf)
+
 
 dispatch_opt = make_dispatcher_method(OptHeap, 'optimize_',
                                       default=OptHeap.emit)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -297,6 +297,7 @@
         self.optrewrite = None
         self.optearlyforce = None
         self.optunroll = None
+        self._really_emitted_operation = None
 
         self._last_guard_op = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -30,6 +30,7 @@
         self.add_guard_future_condition(bridge)
         trace = oparser.convert_loop_to_trace(bridge, FakeMetaInterpStaticData(self.cpu))
         data = compile.BridgeCompileData(trace, self.convert_values(bridge.operations[-1].getarglist(), bridge_values),
+                                         None,
                                          enable_opts=self.enable_opts,
                             inline_short_preamble=inline_short_preamble)
         bridge_info, ops = self._do_optimize_loop(data)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -232,9 +232,15 @@
         return label_vs
 
     def optimize_bridge(self, trace, runtime_boxes, call_pure_results,
-                        inline_short_preamble, box_names_memo):
+                        inline_short_preamble, box_names_memo, resumestorage):
+        from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+        frontend_inputargs = trace.inputargs
         trace = trace.get_iter()
         self._check_no_forwarding([trace.inputargs])
+        if resumestorage:
+            deserialize_optimizer_knowledge(self.optimizer,
+                                            resumestorage, frontend_inputargs,
+                                            trace.inputargs)
         info, ops = self.optimizer.propagate_all_forward(trace,
             call_pure_results, False)
         jump_op = info.jump_op
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -752,7 +752,6 @@
 
     def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)):
         self.setref_base(r)
-        self.datatype = 'r'
 
     def reset_value(self):
         self.setref_base(lltype.nullptr(llmemory.GCREF.TO))
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -162,25 +162,13 @@
 UNINITIALIZED = tag(-2, TAGCONST)   # used for uninitialized string characters
 TAG_CONST_OFFSET = 0
 
-class NumberingState(object):
+class NumberingState(resumecode.Writer):
     def __init__(self, size):
+        resumecode.Writer.__init__(self, size)
         self.liveboxes = {}
-        self.current = [rffi.cast(rffi.SHORT, 0)] * size
-        self._pos = 0
         self.num_boxes = 0
         self.num_virtuals = 0
 
-    def append_short(self, item):
-        self.current[self._pos] = item
-        self._pos += 1
-
-    def append_int(self, item):
-        short = rffi.cast(rffi.SHORT, item)
-        assert rffi.cast(lltype.Signed, short) == item
-        return self.append_short(short)
-
-    def create_numbering(self):
-        return resumecode.create_numbering(self.current)
 
 class ResumeDataLoopMemo(object):
 
@@ -268,6 +256,8 @@
     def number(self, optimizer, position, trace):
         snapshot_iter = trace.get_snapshot_iter(position)
         numb_state = NumberingState(snapshot_iter.size)
+        numb_state.append_int(0) # patch later: size of resume section
+        numb_state.append_int(0) # patch later: number of failargs
 
         arr = snapshot_iter.vable_array
 
@@ -287,6 +277,7 @@
             numb_state.append_int(pc)
             self._number_boxes(
                     snapshot_iter, snapshot.box_array, optimizer, numb_state)
+        numb_state.patch_current_size(0)
 
         return numb_state
 
@@ -471,6 +462,9 @@
         self._number_virtuals(liveboxes, optimizer, num_virtuals)
         self._add_pending_fields(optimizer, pending_setfields)
 
+        numb_state.patch(1, len(liveboxes))
+
+        self._add_optimizer_sections(numb_state, liveboxes, liveboxes_from_env)
         storage.rd_numb = numb_state.create_numbering()
         storage.rd_consts = self.memo.consts
         return liveboxes[:]
@@ -590,6 +584,12 @@
                 return self.liveboxes_from_env[box]
             return self.liveboxes[box]
 
+    def _add_optimizer_sections(self, numb_state, liveboxes, liveboxes_from_env):
+        # add extra information about things the optimizer learned
+        from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge
+        serialize_optimizer_knowledge(
+            self.optimizer, numb_state, liveboxes, liveboxes_from_env, self.memo)
+
 class AbstractVirtualInfo(object):
     kind = REF
     is_about_raw = False
@@ -931,9 +931,10 @@
 
     def _init(self, cpu, storage):
         self.cpu = cpu
-        self.numb = storage.rd_numb
-        self.cur_index = 0
-        self.count = storage.rd_count
+        self.resumecodereader = resumecode.Reader(storage.rd_numb)
+        items_resume_section = self.resumecodereader.next_item()
+        self.items_resume_section = items_resume_section
+        self.count = self.resumecodereader.next_item()
         self.consts = storage.rd_consts
 
     def _prepare(self, storage):
@@ -941,14 +942,21 @@
         self._prepare_pendingfields(storage.rd_pendingfields)
 
     def read_jitcode_pos_pc(self):
-        jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb,
-            self.cur_index)
-        pc, self.cur_index = resumecode.numb_next_item(self.numb,
-            self.cur_index)
+        jitcode_pos = self.resumecodereader.next_item()
+        pc = self.resumecodereader.next_item()
         return jitcode_pos, pc
 
+    def next_int(self):
+        return self.decode_int(self.resumecodereader.next_item())
+
+    def next_ref(self):
+        return self.decode_ref(self.resumecodereader.next_item())
+
+    def next_float(self):
+        return self.decode_float(self.resumecodereader.next_item())
+
     def done_reading(self):
-        return self.cur_index >= len(self.numb.code)
+        return self.resumecodereader.items_read >= self.items_resume_section
 
     def getvirtual_ptr(self, index):
         # Returns the index'th virtual, building it lazily if needed.
@@ -1025,29 +1033,22 @@
     def _prepare_next_section(self, info):
         # Use info.enumerate_vars(), normally dispatching to
         # rpython.jit.codewriter.jitcode.  Some tests give a different 'info'.
-        self.cur_index = info.enumerate_vars(self._callback_i,
-                                        self._callback_r,
-                                        self._callback_f,
-                                        self.unique_id,  # <-- annotation hack
-                                        self.cur_index)
+        info.enumerate_vars(self._callback_i,
+                            self._callback_r,
+                            self._callback_f,
+                            self.unique_id)  # <-- annotation hack
 
-    def _callback_i(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_int(item)
+    def _callback_i(self, register_index):
+        value = self.next_int()
         self.write_an_int(register_index, value)
-        return index
 
-    def _callback_r(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_ref(item)
+    def _callback_r(self, register_index):
+        value = self.next_ref()
         self.write_a_ref(register_index, value)
-        return index
 
-    def _callback_f(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_float(item)
+    def _callback_f(self, register_index):
+        value = self.next_float()
         self.write_a_float(register_index, value)
-        return index
 
 # ---------- when resuming for pyjitpl.py, make boxes ----------
 
@@ -1057,6 +1058,7 @@
     boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info,
                                                       greenfield_info)
     virtualizable_boxes, virtualref_boxes = boxes
+
     while not resumereader.done_reading():
         jitcode_pos, pc = resumereader.read_jitcode_pos_pc()
         jitcode = metainterp.staticdata.jitcodes[jitcode_pos]
@@ -1076,7 +1078,7 @@
         self._init(metainterp.cpu, storage)
         self.deadframe = deadframe
         self.metainterp = metainterp
-        self.liveboxes = [None] * storage.rd_count
+        self.liveboxes = [None] * self.count
         self._prepare(storage)
 
     def consume_boxes(self, info, boxes_i, boxes_r, boxes_f):
@@ -1085,42 +1087,30 @@
         self.boxes_f = boxes_f
         self._prepare_next_section(info)
 
-    def consume_virtualizable_boxes(self, vinfo, index):
+    def consume_virtualizable_boxes(self, vinfo):
         # we have to ignore the initial part of 'nums' (containing vrefs),
         # find the virtualizable from nums[-1], and use it to know how many
         # boxes of which type we have to return.  This does not write
         # anything into the virtualizable.
-        numb = self.numb
-        item, index = resumecode.numb_next_item(numb, index)
-        virtualizablebox = self.decode_ref(item)
+        virtualizablebox = self.next_ref()
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox)
-        return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox,
-            numb, index)
+        return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox)
 
-    def consume_virtualref_boxes(self, index):
+    def consume_virtualref_boxes(self):
         # Returns a list of boxes, assumed to be all BoxPtrs.
         # We leave up to the caller to call vrefinfo.continue_tracing().
-        size, index = resumecode.numb_next_item(self.numb, index)
-        if size == 0:
-            return [], index
-        lst = []
-        for i in range(size * 2):
-            item, index = resumecode.numb_next_item(self.numb, index)
-            lst.append(self.decode_ref(item))
-        return lst, index
+        size = self.resumecodereader.next_item()
+        return [self.next_ref() for i in range(size * 2)]
 
     def consume_vref_and_vable_boxes(self, vinfo, ginfo):
-        vable_size, index = resumecode.numb_next_item(self.numb, 0)
+        vable_size = self.resumecodereader.next_item()
         if vinfo is not None:
-            virtualizable_boxes, index = self.consume_virtualizable_boxes(vinfo,
-                                                                          index)
+            virtualizable_boxes = self.consume_virtualizable_boxes(vinfo)
         elif ginfo is not None:
-            item, index = resumecode.numb_next_item(self.numb, index)
-            virtualizable_boxes = [self.decode_ref(item)]
+            virtualizable_boxes = [self.next_ref()]
         else:
             virtualizable_boxes = None
-        virtualref_boxes, index = self.consume_virtualref_boxes(index)
-        self.cur_index = index
+        virtualref_boxes = self.consume_virtualref_boxes()
         return virtualizable_boxes, virtualref_boxes
 
     def allocate_with_vtable(self, descr=None):
@@ -1297,7 +1287,7 @@
         self.liveboxes[num] = box
         return box
 
-    def decode_box_of_type(self, TYPE, tagged):
+    def next_box_of_type(self, TYPE):
         kind = getkind(TYPE)
         if kind == 'int':
             kind = INT
@@ -1307,8 +1297,8 @@
             kind = FLOAT
         else:
             raise AssertionError(kind)
-        return self.decode_box(tagged, kind)
-    decode_box_of_type._annspecialcase_ = 'specialize:arg(1)'
+        return self.decode_box(self.resumecodereader.next_item(), kind)
+    next_box_of_type._annspecialcase_ = 'specialize:arg(1)'
 
     def write_an_int(self, index, box):
         self.boxes_i[index] = box
@@ -1397,64 +1387,54 @@
         info = blackholeinterp.get_current_position_info()
         self._prepare_next_section(info)
 
-    def consume_virtualref_info(self, vrefinfo, index):
+    def consume_virtualref_info(self, vrefinfo):
         # we have to decode a list of references containing pairs
         # [..., virtual, vref, ...] and returns the index at the end
-        size, index = resumecode.numb_next_item(self.numb, index)
+        size = self.resumecodereader.next_item()
         if vrefinfo is None or size == 0:
             assert size == 0
-            return index
+            return
         for i in range(size):
-            virtual_item, index = resumecode.numb_next_item(
-                self.numb, index)
-            vref_item, index = resumecode.numb_next_item(
-                self.numb, index)
-            virtual = self.decode_ref(virtual_item)
-            vref = self.decode_ref(vref_item)
+            virtual = self.next_ref()
+            vref = self.next_ref()
             # For each pair, we store the virtual inside the vref.
             vrefinfo.continue_tracing(vref, virtual)
-        return index
 
-    def consume_vable_info(self, vinfo, index):
+    def consume_vable_info(self, vinfo):
         # we have to ignore the initial part of 'nums' (containing vrefs),
         # find the virtualizable from nums[-1], load all other values
         # from the CPU stack, and copy them into the virtualizable
-        numb = self.numb
-        item, index = resumecode.numb_next_item(self.numb, index)
-        virtualizable = self.decode_ref(item)
+        virtualizable = self.next_ref()
         # just reset the token, we'll force it later
         vinfo.reset_token_gcref(virtualizable)
-        index = vinfo.write_from_resume_data_partial(virtualizable, self,
-            index, numb)
-        return index
+        vinfo.write_from_resume_data_partial(virtualizable, self)
 
-    def load_value_of_type(self, TYPE, tagged):
+    def load_next_value_of_type(self, TYPE):
         from rpython.jit.metainterp.warmstate import specialize_value
         kind = getkind(TYPE)
         if kind == 'int':
-            x = self.decode_int(tagged)
+            x = self.next_int()
         elif kind == 'ref':
-            x = self.decode_ref(tagged)
+            x = self.next_ref()
         elif kind == 'float':
-            x = self.decode_float(tagged)
+            x = self.next_float()
         else:
             raise AssertionError(kind)
         return specialize_value(TYPE, x)
-    load_value_of_type._annspecialcase_ = 'specialize:arg(1)'
+    load_next_value_of_type._annspecialcase_ = 'specialize:arg(1)'
 
     def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo):
-        vable_size, index = resumecode.numb_next_item(self.numb, 0)
+        vable_size = self.resumecodereader.next_item()
         if self.resume_after_guard_not_forced != 2:
             if vinfo is not None:
-                index = self.consume_vable_info(vinfo, index)
+                self.consume_vable_info(vinfo)
             if ginfo is not None:
-                _, index = resumecode.numb_next_item(self.numb, index)
-            index = self.consume_virtualref_info(vrefinfo, index)
+                _ = self.resumecodereader.next_item()
+            self.consume_virtualref_info(vrefinfo)
         else:
-            index = resumecode.numb_next_n_items(self.numb, vable_size, index)
-            vref_size, index = resumecode.numb_next_item(self.numb, index)
-            index = resumecode.numb_next_n_items(self.numb, vref_size * 2, index)
-        self.cur_index = index 
+            self.resumecodereader.jump(vable_size)
+            vref_size = self.resumecodereader.next_item()
+            self.resumecodereader.jump(vref_size * 2)
 
     def allocate_with_vtable(self, descr=None):
         from rpython.jit.metainterp.executor import exec_new_with_vtable
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -1,6 +1,9 @@
 
 """ Resume bytecode. It goes as following:
 
+  # ----- resume section
+  [total size of resume section]
+  [number of failargs]
   [<length> <virtualizable object> <numb> <numb> <numb>]    if vinfo is not None
    -OR-
   [1 <ginfo object>]                                        if ginfo is not None
@@ -13,10 +16,14 @@
   [<pc> <jitcode> <numb> <numb>]
   ...
 
-  until the length of the array.
+  until the size of the resume section
+
+  # ----- optimization section
+  <more code>                                      further sections according to bridgeopt.py
 """
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib import objectmodel
 
 NUMBERINGP = lltype.Ptr(lltype.GcForwardReference())
 NUMBERING = lltype.GcStruct('Numbering',
@@ -24,33 +31,24 @@
 NUMBERINGP.TO.become(NUMBERING)
 NULL_NUMBER = lltype.nullptr(NUMBERING)
 
-def create_numbering(lst, total=-1):
-    if total == -1:
-        total = len(lst)
-    result = []
-    for i in range(total):
-        item = lst[i]
-        item = rffi.cast(lltype.Signed, item)
-        item *= 2
-        if item < 0:
-            item = -1 - item
+def append_numbering(lst, item):
+    item = rffi.cast(lltype.Signed, item)
+    item *= 2
+    if item < 0:
+        item = -1 - item
 
-        assert item >= 0
-        if item < 2**7:
-            result.append(rffi.cast(rffi.UCHAR, item))
-        elif item < 2**14:
-            result.append(rffi.cast(rffi.UCHAR, item | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, item >> 7))
-        else:
-            assert item < 2**16
-            result.append(rffi.cast(rffi.UCHAR, item | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, item >> 14))
+    assert item >= 0
+    if item < 2**7:
+        lst.append(rffi.cast(rffi.UCHAR, item))
+    elif item < 2**14:
+        lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, item >> 7))
+    else:
+        assert item < 2**16
+        lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, item >> 14))
 
-    numb = lltype.malloc(NUMBERING, len(result))
-    for i in range(len(result)):
-        numb.code[i] = result[i]
-    return numb
 
 def numb_next_item(numb, index):
     value = rffi.cast(lltype.Signed, numb.code[index])
@@ -81,3 +79,64 @@
         next, i = numb_next_item(numb, i)
         l.append(next)
     return l
+
+class Writer(object):
+    def __init__(self, size=0):
+        self.current = objectmodel.newlist_hint(size)
+
+    def append_short(self, item):
+        self.current.append(item)
+
+    def append_int(self, item):
+        short = rffi.cast(rffi.SHORT, item)
+        assert rffi.cast(lltype.Signed, short) == item
+        return self.append_short(short)
+
+    def create_numbering(self):
+        final = objectmodel.newlist_hint(len(self.current) * 3)
+        for item in self.current:
+            append_numbering(final, item)
+        numb = lltype.malloc(NUMBERING, len(final))
+        for i, elt in enumerate(final):
+            numb.code[i] = elt
+        return numb
+
+    def patch_current_size(self, index):
+        self.patch(index, len(self.current))
+
+    def patch(self, index, item):
+        self.current[index] = item
+
+def create_numbering(l):
+    w = Writer()
+    for item in l:
+        w.append_int(item)
+    return w.create_numbering()
+
+
+class Reader(object):
+    def __init__(self, code):
+        self.code = code
+        self.cur_pos = 0 # index into the code
+        self.items_read = 0 # number of items read
+
+    def next_item(self):
+        result, self.cur_pos = numb_next_item(self.code, self.cur_pos)
+        self.items_read += 1
+        return result
+
+    def peek(self):
+        result, _ = numb_next_item(self.code, self.cur_pos)
+        return result
+
+    def jump(self, size):
+        """ jump n items forward without returning anything """
+        index = self.cur_pos
+        for i in range(size):
+            _, index = numb_next_item(self.code, index)
+        self.items_read += size
+        self.cur_pos = index
+
+    def unpack(self):
+        # mainly for debugging
+        return unpack_numbering(self.code)
diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_bridgeopt.py
@@ -0,0 +1,188 @@
+# tests that check that information is fed from the optimizer into the bridges
+
+import math
+from rpython.rlib import jit
+from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge
+from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+from rpython.jit.metainterp.resoperation import InputArgRef, InputArgInt
+from rpython.jit.metainterp.resume import NumberingState
+from rpython.jit.metainterp.resumecode import unpack_numbering
+from rpython.jit.metainterp.optimizeopt.info import InstancePtrInfo
+
+from hypothesis import strategies, given
+
+class FakeTS(object):
+    def __init__(self, dct):
+        self.dct = dct
+
+    def cls_of_box(self, box):
+        return self.dct[box]
+
+
+class FakeCPU(object):
+    def __init__(self, dct):
+        self.ts = FakeTS(dct)
+
+class FakeOptimizer(object):
+    metainterp_sd = None
+    optheap = None
+
+    def __init__(self, dct={}, cpu=None):
+        self.dct = dct
+        self.constant_classes = {}
+        self.cpu = cpu
+
+    def getptrinfo(self, arg):
+        return self.dct.get(arg, None)
+
+    def make_constant_class(self, arg, cls):
+        self.constant_classes[arg] = cls
+
+class FakeClass(object):
+    pass
+
+class FakeStorage(object):
+    def __init__(self, numb):
+        self.rd_numb = numb
+
+def test_known_classes():
+    box1 = InputArgRef()
+    box2 = InputArgRef()
+    box3 = InputArgRef()
+
+    cls = FakeClass()
+    dct = {box1: InstancePtrInfo(known_class=cls)}
+    optimizer = FakeOptimizer(dct)
+
+    numb_state = NumberingState(4)
+    numb_state.append_int(1) # size of resume block
+    liveboxes = [InputArgInt(), box2, box1, box3]
+
+    serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
+
+    assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0]
+
+    rbox1 = InputArgRef()
+    rbox2 = InputArgRef()
+    rbox3 = InputArgRef()
+    after_optimizer = FakeOptimizer(cpu=FakeCPU({rbox1: cls}))
+    deserialize_optimizer_knowledge(
+        after_optimizer, FakeStorage(numb_state.create_numbering()),
+        [InputArgInt(), rbox2, rbox1, rbox3], liveboxes)
+    assert box1 in after_optimizer.constant_classes
+    assert box2 not in after_optimizer.constant_classes
+    assert box3 not in after_optimizer.constant_classes
+
+
+box_strategy = strategies.builds(InputArgInt) | strategies.builds(InputArgRef)
+tuples = strategies.tuples(box_strategy, strategies.booleans()).filter(
+        lambda (box, known_class): isinstance(box, InputArgRef) or not known_class)
+boxes_known_classes = strategies.lists(tuples, min_size=1)
+
+ at given(boxes_known_classes)
+def test_random_class_knowledge(boxes_known_classes):
+    cls = FakeClass()
+    dct1 = {box: InstancePtrInfo(known_class=cls)
+              for box, known_class in boxes_known_classes
+                  if known_class}
+    optimizer = FakeOptimizer(dct1)
+
+    refboxes = [box for (box, _) in boxes_known_classes
+                    if isinstance(box, InputArgRef)]
+
+    numb_state = NumberingState(1)
+    numb_state.append_int(1) # size of resume block
+    liveboxes = [box for (box, _) in boxes_known_classes]
+
+    serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
+
+    assert len(numb_state.create_numbering().code) == 2 + math.ceil(len(refboxes) / 6.0)
+
+    dct = {box: cls
+              for box, known_class in boxes_known_classes
+                  if known_class}
+    after_optimizer = FakeOptimizer(cpu=FakeCPU(dct))
+    deserialize_optimizer_knowledge(
+        after_optimizer, FakeStorage(numb_state.create_numbering()),
+        liveboxes, liveboxes)
+    for box, known_class in boxes_known_classes:
+        assert (box in after_optimizer.constant_classes) == known_class
+
+class TestOptBridge(LLJitMixin):
+    # integration tests
+    def test_bridge_guard_class(self):
+        myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a'])
+        class A(object):
+            def f(self):
+                return 1
+        class B(A):
+            def f(self):
+                return 2
+        def f(x, y, n):
+            if x:
+                a = A()
+            else:
+                a = B()
+            a.x = 0
+            res = 0
+            while y > 0:
+                myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a)
+                res += a.f()
+                a.x += 1
+                if y > n:
+                    res += 1
+                res += a.f()
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 32, 16])
+        assert res == f(6, 32, 16)
+        self.check_trace_count(3)
+        self.check_resops(guard_class=1)
+
+    def test_bridge_field_read(self):
+        myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a'])
+        class A(object):
+            def f(self):
+                return 1
+        class B(A):
+            def f(self):
+                return 2
+        class M(object):
+            _immutable_fields_ = ['x']
+            def __init__(self, x):
+                self.x = x
+
+        m1 = M(1)
+        m2 = M(2)
+        def f(x, y, n):
+            if x:
+                a = A()
+                a.m = m1
+                a.n = n
+            else:
+                a = B()
+                a.m = m2
+                a.n = n
+            a.x = 0
+            res = 0
+            while y > 0:
+                myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a)
+                n1 = a.n
+                m = jit.promote(a.m)
+                res += m.x
+                a.x += 1
+                if y > n:
+                    res += 1
+                m = jit.promote(a.m)
+                res += m.x
+                res += n1 + a.n
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 32, 16])
+        assert res == f(6, 32, 16)
+        self.check_trace_count(3)
+        self.check_resops(guard_value=1)
+        self.check_resops(getfield_gc_i=4) # 3x a.x, 1x a.n
+        self.check_resops(getfield_gc_r=1) # in main loop
+
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -79,6 +79,7 @@
 def test_compile_loop():
     cpu = FakeCPU()
     staticdata = FakeMetaInterpStaticData()
+    staticdata.all_descrs = LLtypeMixin.cpu.setup_descrs()
     staticdata.cpu = cpu
     staticdata.jitlog = jl.JitLogger(cpu)
     staticdata.jitlog.trace_id = 1
diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py
--- a/rpython/jit/metainterp/test/test_greenfield.py
+++ b/rpython/jit/metainterp/test/test_greenfield.py
@@ -49,7 +49,7 @@
         #
         res = self.meta_interp(g, [7])
         assert res == -22
-        self.check_trace_count(6)
+        self.check_trace_count(4)
         self.check_resops(guard_value=0)
 
     def test_green_field_3(self):
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -36,10 +36,12 @@
     rd_consts = []
     rd_virtuals = None
     rd_pendingfields = None
-    rd_count = 0
 
 
 class FakeOptimizer(object):
+    metainterp_sd = None
+    optheap = None
+
     def __init__(self, trace=None):
         self.trace = trace
 
@@ -251,18 +253,17 @@
     def get_current_position_info(self):
         class MyInfo:
             @staticmethod
-            def enumerate_vars(callback_i, callback_r, callback_f, _, index):
+            def enumerate_vars(callback_i, callback_r, callback_f, _):
                 count_i = count_r = count_f = 0
                 for ARG in self.ARGS:
                     if ARG == lltype.Signed:
-                        index = callback_i(index, count_i); count_i += 1
+                        callback_i(count_i); count_i += 1
                     elif ARG == llmemory.GCREF:
-                        index = callback_r(index, count_r); count_r += 1
+                        callback_r(count_r); count_r += 1
                     elif ARG == longlong.FLOATSTORAGE:
-                        index = callback_f(index, count_f); count_f += 1
+                        callback_f(count_f); count_f += 1
                     else:
                         assert 0
-                return index
         return MyInfo()
 
     def setarg_i(self, index, value):
@@ -289,7 +290,8 @@
     assert bh.written_f == expected_f
 
 
-Numbering = create_numbering
+def Numbering(l):
+    return create_numbering([len(l)] + l) # prefix index to the end of thing
 
 def tagconst(i):
     return tag(i + TAG_CONST_OFFSET, TAGCONST)
@@ -299,12 +301,11 @@
     c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)]
     storage = Storage()
     storage.rd_consts = [c1, c2, c3]
-    numb = Numbering([tag(0, TAGBOX), tagconst(0),
+    numb = Numbering([3, tag(0, TAGBOX), tagconst(0),
                        NULLREF, tag(0, TAGBOX), tag(1, TAGBOX)] +
-                       [tagconst(1), tagconst(2)] + 
+                       [tagconst(1), tagconst(2)] +
                        [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)])
     storage.rd_numb = numb
-    storage.rd_count = 3
     #
     cpu = MyCPU([42, gcref1, -66])
     metainterp = MyMetaInterp(cpu)
@@ -345,7 +346,7 @@
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
-    numb = Numbering([tag(100, TAGINT)])
+    numb = Numbering([1, tag(100, TAGINT)])
     storage.rd_numb = numb
     #
     cpu = MyCPU([])
@@ -362,10 +363,9 @@
             return s
     class FakeStorage(object):
         rd_virtuals = [FakeVinfo(), None]
-        rd_numb = []
+        rd_numb = Numbering([1])
         rd_consts = []
         rd_pendingfields = None
-        rd_count = 0
     class FakeMetainterp(object):
         _already_allocated_resume_virtuals = None
         cpu = None
@@ -773,12 +773,12 @@
     assert untag(tagged) == (44, TAGINT)
     tagged = memo.getconst(ConstInt(-3))
     assert untag(tagged) == (-3, TAGINT)
-    const = ConstInt(50000)
+    const = ConstInt(5000000)
     tagged = memo.getconst(const)
     index, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert memo.consts[index - TAG_CONST_OFFSET] is const
-    tagged = memo.getconst(ConstInt(50000))
+    tagged = memo.getconst(ConstInt(5000000))
     index2, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert index2 == index
@@ -858,7 +858,7 @@
     base = [0, 0, tag(0, TAGBOX), tag(1, TAGINT),
             tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)]
 
-    assert unpack_numbering(numb) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                       tag(0, TAGBOX), tag(1, TAGINT)]
     t.append(0)
     snap2 = t.create_top_snapshot(FakeJitCode("jitcode", 0), 2, Frame(env2),
@@ -872,7 +872,7 @@
     assert numb_state2.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b3: tag(2, TAGBOX)}
     assert numb_state2.liveboxes is not numb_state.liveboxes
-    assert unpack_numbering(numb2) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb2) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                        tag(0, TAGBOX), tag(3, TAGINT)]
 
     t.append(0)
@@ -894,7 +894,7 @@
     assert numb_state3.num_virtuals == 0
     
     assert numb_state3.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
-    assert unpack_numbering(numb3) == ([0, 2, tag(3, TAGINT), tag(4, TAGINT),
+    assert unpack_numbering(numb3) == ([17, 0, 0, 2, tag(3, TAGINT), tag(4, TAGINT),
                                        tag(0, TAGBOX), tag(3, TAGINT)] +
                                        base + [0, 2])
 
@@ -911,7 +911,7 @@
     
     assert numb_state4.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL)}
-    assert unpack_numbering(numb4) == [0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
+    assert unpack_numbering(numb4) == [17, 0, 0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
                                        tag(0, TAGBOX), tag(3, TAGINT)] + base + [0, 2]
 
     t.append(0)
@@ -930,7 +930,7 @@
 
     assert numb_state5.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)}
-    assert unpack_numbering(numb5) == [
+    assert unpack_numbering(numb5) == [22, 0,
         3, tag(0, TAGBOX), tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL),
         0] + base + [
         2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT)
@@ -949,15 +949,17 @@
     numb_state = memo.number(FakeOptimizer(), 0, i)
     numb = numb_state.create_numbering()
     l = unpack_numbering(numb)
-    assert l[0] == 0
+    assert l[0] == len(l)
+    assert l[1] == 0
     assert l[1] == 0
     assert l[2] == 0
     assert l[3] == 0
+    assert l[4] == 0
     mapping = dict(zip(inpargs, i.inputargs))
     for i, item in enumerate(lst):
-        v, tag = untag(l[i + 4])
+        v, tag = untag(l[i + 6])
         if tag == TAGBOX:
-            assert l[i + 4] == numb_state.liveboxes[mapping[item]]
+            assert l[i + 6] == numb_state.liveboxes[mapping[item]]
         elif tag == TAGCONST:
             assert memo.consts[v].getint() == item.getint()
         elif tag == TAGINT:
@@ -1069,15 +1071,15 @@
     cpu = MyCPU([])
     reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe")
     reader.consume_vref_and_vable(None, None, None)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, sys.maxint, 1, sys.maxint, 2**16)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, 2, 3)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, sys.maxint, 2**16, -65)
 
 def test_virtual_adder_memo_const_sharing():
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**23), ConstInt(-65)]
     storage, t = make_storage(b1s, b2s, b3s)
     metainterp_sd = FakeMetaInterpStaticData()
     memo = ResumeDataLoopMemo(metainterp_sd)
@@ -1087,7 +1089,7 @@
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**24), ConstInt(-65)]
     storage2, t = make_storage(b1s, b2s, b3s)
     i = t.get_iter()
     modifier2 = ResumeDataVirtualAdder(FakeOptimizer(i), storage2, storage2,
@@ -1112,9 +1114,10 @@
                 return True
         class MyInfo:
             @staticmethod
-            def enumerate_vars(callback_i, callback_r, callback_f, _, index):
-                while index < len(self.numb.code):
-                    tagged, _ = resumecode.numb_next_item(self.numb, index)
+            def enumerate_vars(callback_i, callback_r, callback_f, _):
+                index = 0
+                while not self.done_reading():
+                    tagged = self.resumecodereader.peek()
                     _, tag = untag(tagged)
                     if tag == TAGVIRTUAL:
                         kind = REF
@@ -1122,20 +1125,21 @@
                         kind = Whatever()
                     box = self.decode_box(tagged, kind)
                     if box.type == INT:
-                        index = callback_i(index, index)
+                        callback_i(index)
                     elif box.type == REF:
-                        index = callback_r(index, index)
+                        callback_r(index)
                     elif box.type == FLOAT:
-                        index = callback_f(index, index)
+                        callback_f(index)
                     else:
                         assert 0
+                    index += 1
 
-        size, self.cur_index = resumecode.numb_next_item(self.numb, 0)
+        size = self.resumecodereader.next_item()
         assert size == 0
-        size, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
+        size = self.resumecodereader.next_item()
         assert size == 0
-        pc, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
-        jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
+        pc = self.resumecodereader.next_item()
+        jitcode_pos = self.resumecodereader.next_item()
 
         self._prepare_next_section(MyInfo())
         return self.lst
@@ -1228,7 +1232,7 @@
     liveboxes = []
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     b3t, b5t = [IntFrontendOp(0), RefFrontendOp(0)]
     b5t.setref_base(demo55o)
@@ -1299,7 +1303,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     b1t, b3t, b4t = [IntFrontendOp(0), IntFrontendOp(0), IntFrontendOp(0)]
     b1t.setint(11)
@@ -1352,7 +1356,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     b4t = RefFrontendOp(0)
     newboxes = _resume_remap(liveboxes, [#b2s -- virtual
                                          b4s], b4t)
@@ -1398,7 +1402,7 @@
     modifier._add_pending_fields(FakeOptimizer(), [
         ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)])
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     demo55.next = lltype.nullptr(LLtypeMixin.NODE)
     b2t = RefFrontendOp(0)
diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -1,29 +1,62 @@
-
-from rpython.jit.metainterp.resumecode import NUMBERING, NULL_NUMBER
 from rpython.jit.metainterp.resumecode import create_numbering,\
-    unpack_numbering
+    unpack_numbering, Reader, Writer
 from rpython.rtyper.lltypesystem import lltype
 
-from hypothesis import strategies, given
+from hypothesis import strategies, given, example
 
+examples = [
+    [1, 2, 3, 4, 257, 10000, 13, 15],
+    [1, 2, 3, 4],
+    range(1, 10, 2),
+    [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
+]
 
-def test_pack_unpack():
-    examples = [
-        [1, 2, 3, 4, 257, 10000, 13, 15],
-        [1, 2, 3, 4],
-        range(1, 10, 2),
-        [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
-    ]
-    for l in examples:
-        n = create_numbering(l)
-        assert unpack_numbering(n) == l
+def hypothesis_and_examples(func):
+    func = given(strategies.lists(strategies.integers(-2**15, 2**15-1)))(func)
+    for ex in examples:
+        func = example(ex)(func)
+    return func
 
- at given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+ at hypothesis_and_examples
 def test_roundtrip(l):
     n = create_numbering(l)
     assert unpack_numbering(n) == l
 
- at given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+ at hypothesis_and_examples
 def test_compressing(l):
     n = create_numbering(l)
     assert len(n.code) <= len(l) * 3
+
+ at hypothesis_and_examples
+def test_reader(l):
+    n = create_numbering(l)
+    r = Reader(n)
+    for i, elt in enumerate(l):
+        assert r.items_read == i
+        item = r.next_item()
+        assert elt == item
+
+ at hypothesis_and_examples
+def test_writer(l):
+    for size in [len(l), 0]:
+        w = Writer(len(l))
+        for num in l:
+            w.append_int(num)
+        n = w.create_numbering()
+        assert unpack_numbering(n) == l
+
+ at hypothesis_and_examples
+def test_patch(l):
+    for middle in range(len(l)):
+        l1 = l[:middle]
+        l2 = l[middle:]
+        w = Writer(len(l))
+        w.append_int(0)
+        for num in l1:
+            w.append_int(num)
+        w.patch_current_size(0)
+        for num in l2:
+            w.append_int(num)
+        n = w.create_numbering()
+        assert unpack_numbering(n)[1:] == l
+        assert unpack_numbering(n)[0] == middle + 1
diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -2,7 +2,6 @@
 from rpython.jit.metainterp import history
 from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem
 from rpython.jit.metainterp.warmstate import wrap, unwrap
-from rpython.jit.metainterp.resumecode import numb_next_item
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import rvirtualizable
 from rpython.rtyper.lltypesystem import lltype, llmemory
@@ -127,24 +126,21 @@
                 size += 1
             return size
 
-        def write_from_resume_data_partial(virtualizable, reader, index, numb):
+        def write_from_resume_data_partial(virtualizable, reader):
             virtualizable = cast_gcref_to_vtype(virtualizable)
             # Load values from the reader (see resume.py) described by
             # the list of numbers 'nums', and write them in their proper
             # place in the 'virtualizable'.
             for FIELDTYPE, fieldname in unroll_static_fields:
-                item, index = numb_next_item(numb, index)
-                x = reader.load_value_of_type(FIELDTYPE, item)
+                x = reader.load_next_value_of_type(FIELDTYPE)
                 setattr(virtualizable, fieldname, x)
             for ARRAYITEMTYPE, fieldname in unroll_array_fields:
                 lst = getattr(virtualizable, fieldname)
                 for j in range(getlength(lst)):
-                    item, index = numb_next_item(numb, index)                    
-                    x = reader.load_value_of_type(ARRAYITEMTYPE, item)
+                    x = reader.load_next_value_of_type(ARRAYITEMTYPE)
                     setarrayitem(lst, j, x)
-            return index
 
-        def load_list_of_boxes(virtualizable, reader, vable_box, numb, index):
+        def load_list_of_boxes(virtualizable, reader, vable_box):
             virtualizable = cast_gcref_to_vtype(virtualizable)
             # Uses 'virtualizable' only to know the length of the arrays;
             # does not write anything into it.  The returned list is in
@@ -152,17 +148,15 @@
             # the virtualizable itself.
             boxes = []
             for FIELDTYPE, fieldname in unroll_static_fields:
-                item, index = numb_next_item(numb, index)
-                box = reader.decode_box_of_type(FIELDTYPE, item)
+                box = reader.next_box_of_type(FIELDTYPE)
                 boxes.append(box)
             for ARRAYITEMTYPE, fieldname in unroll_array_fields:
                 lst = getattr(virtualizable, fieldname)
                 for j in range(getlength(lst)):
-                    item, index = numb_next_item(numb, index)                    
-                    box = reader.decode_box_of_type(ARRAYITEMTYPE, item)
+                    box = reader.next_box_of_type(ARRAYITEMTYPE)
                     boxes.append(box)
             boxes.append(vable_box)
-            return boxes, index
+            return boxes
 
         def check_boxes(virtualizable, boxes):
             virtualizable = cast_gcref_to_vtype(virtualizable)

From pypy.commits at gmail.com  Sun Feb 26 06:32:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 26 Feb 2017 03:32:12 -0800 (PST)
Subject: [pypy-commit] pypy default: Oops, fix translation
Message-ID: <58b2bcbc.e5d7190a.14a05.fb71@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90359:78cdf1e8c131
Date: 2017-02-26 12:31 +0100
http://bitbucket.org/pypy/pypy/changeset/78cdf1e8c131/

Log:	Oops, fix translation

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -70,7 +70,7 @@
     def same_info(self, other):
         return self is other
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
         return None
 
     def getstrhash(self, op, mode):
@@ -777,21 +777,20 @@
             # XXX we can do better if we know it's an array
             return IntLowerBound(0)
         else:
-            return ConstIntBound(self.getstrlen(None, None, mode).getint())
-    
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+            return ConstIntBound(self.getstrlen1(mode))
+
+    def getstrlen(self, op, string_optimizer, mode):
+        return ConstInt(self.getstrlen1(mode))
+
+    def getstrlen1(self, mode):
         from rpython.jit.metainterp.optimizeopt import vstring
         
         if mode is vstring.mode_string:
             s = self._unpack_str(vstring.mode_string)
-            if s is None:
-                return None
-            return ConstInt(len(s))
+            return len(s)
         else:
             s = self._unpack_str(vstring.mode_unicode)            
-            if s is None:
-                return None
-            return ConstInt(len(s))
+            return len(s)
 
     def getstrhash(self, op, mode):
         from rpython.jit.metainterp.optimizeopt import vstring
@@ -812,7 +811,7 @@
         from rpython.jit.metainterp.optimizeopt import vstring
         from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
 
-        lgt = self.getstrlen(op, string_optimizer, mode, False)
+        lgt = self.getstrlen(op, string_optimizer, mode)
         return vstring.copy_str_content(string_optimizer, self._const,
                                         targetbox, CONST_0, offsetbox,
                                         lgt, mode)

From pypy.commits at gmail.com  Sun Feb 26 06:44:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 26 Feb 2017 03:44:26 -0800 (PST)
Subject: [pypy-commit] pypy default: update the faq about the gil
Message-ID: <58b2bf9a.4f5c190a.1551b.77df@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90360:75edc48a7774
Date: 2017-02-26 12:43 +0100
http://bitbucket.org/pypy/pypy/changeset/75edc48a7774/

Log:	update the faq about the gil

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -156,19 +156,30 @@
 Does PyPy have a GIL?  Why?
 -------------------------------------------------
 
-Yes, PyPy has a GIL.  Removing the GIL is very hard.  The problems are
-essentially the same as with CPython (including the fact that our
-garbage collectors are not thread-safe so far).  Fixing it is possible,
-as shown by Jython and IronPython, but difficult.  It would require
-adapting the whole source code of PyPy, including subtle decisions about
-whether some effects are ok or not for the user (i.e. the Python
-programmer).
+Yes, PyPy has a GIL.  Removing the GIL is very hard.  On top of CPython,
+you have two problems:  (1) GC, in this case reference counting; (2) the
+whole Python language.
 
-Instead, since 2012, there is work going on on a still very experimental
-:doc:`Software Transactional Memory <stm>` (STM) version of PyPy.  This should give
-an alternative PyPy which works without a GIL, while at the same time
-continuing to give the Python programmer the complete illusion of having
-one.
+For PyPy, the hard issue is (2): by that I mean issues like what occurs
+if a mutable object is changed from one thread and read from another
+concurrently.  This is a problem for *any* mutable type: it needs
+careful review and fixes (fine-grained locks, mostly) through the
+*whole* Python interpreter.  It is a major effort, although not
+completely impossible, as Jython/IronPython showed.  This includes
+subtle decisions about whether some effects are ok or not for the user
+(i.e. the Python programmer).
+
+CPython has additionally the problem (1) of reference counting.  With
+PyPy, the problem is simpler: we need to make our GC multithread-aware.
+This is easier to do efficiently in PyPy than in CPython.  It doesn't
+solve the issue (2), though.
+
+Note that since 2012 there is work going on on a still very experimental
+:doc:`Software Transactional Memory <stm>` version of PyPy.  This should
+give an alternative PyPy which works without a GIL, while at the same
+time continuing to give the Python programmer the complete illusion of
+having one.  This work is currently a bit stalled because of its own
+technical difficulties.
 
 
 Is PyPy more clever than CPython about Tail Calls?

From pypy.commits at gmail.com  Sun Feb 26 06:45:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 26 Feb 2017 03:45:39 -0800 (PST)
Subject: [pypy-commit] pypy default: tweaks
Message-ID: <58b2bfe3.1e142e0a.57aef.f69c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90361:d696e7a41931
Date: 2017-02-26 12:44 +0100
http://bitbucket.org/pypy/pypy/changeset/d696e7a41931/

Log:	tweaks

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -170,15 +170,15 @@
 (i.e. the Python programmer).
 
 CPython has additionally the problem (1) of reference counting.  With
-PyPy, the problem is simpler: we need to make our GC multithread-aware.
-This is easier to do efficiently in PyPy than in CPython.  It doesn't
-solve the issue (2), though.
+PyPy, this sub-problem is simpler: we need to make our GC
+multithread-aware.  This is easier to do efficiently in PyPy than in
+CPython.  It doesn't solve the issue (2), though.
 
 Note that since 2012 there is work going on on a still very experimental
-:doc:`Software Transactional Memory <stm>` version of PyPy.  This should
-give an alternative PyPy which works without a GIL, while at the same
-time continuing to give the Python programmer the complete illusion of
-having one.  This work is currently a bit stalled because of its own
+:doc:`Software Transactional Memory <stm>` (STM) version of PyPy.  This
+should give an alternative PyPy which works without a GIL, while at the
+same time continuing to give the Python programmer the complete illusion
+of having one.  This work is currently a bit stalled because of its own
 technical difficulties.
 
 

From pypy.commits at gmail.com  Sun Feb 26 08:45:30 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sun, 26 Feb 2017 05:45:30 -0800 (PST)
Subject: [pypy-commit] stmgc default: Update gcc-seg-gs instructions.
Message-ID: <58b2dbfa.51d2190a.bc751.fdef@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: 
Changeset: r2014:ee3e379fac1e
Date: 2017-02-26 14:46 +0100
http://bitbucket.org/pypy/stmgc/changeset/ee3e379fac1e/

Log:	Update gcc-seg-gs instructions.

	Users still need to disable some buggy GCC passes.

	GCC 7.x fixes some of these bugs, but there are still at least two
	unresolved bugs (links at the bottom of this commit). The
	instructions will be updated again when GCC 7.1 is released,
	depending on whether these bugs are fixed or not.

	https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66768
	https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79690

diff --git a/gcc-seg-gs/README.txt b/gcc-seg-gs/README.txt
--- a/gcc-seg-gs/README.txt
+++ b/gcc-seg-gs/README.txt
@@ -8,9 +8,8 @@
 compile the standard gcc.  Of course, it is likely that gcc 6.1 will
 soon be available from your Linux distribution directly.
 
-Note that with gcc 6.1, you no longer need gcc-5.1.0-patch.diff, and you
-should not need the "-fno-*" options either (but we didn't check that
-yet).
+Note that with gcc 6.1, you no longer need gcc-5.1.0-patch.diff, but you
+still need the "-fno-*" options.
 
 
 

From pypy.commits at gmail.com  Sun Feb 26 08:58:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 26 Feb 2017 05:58:30 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix issue 2289 (hopefully)
Message-ID: <58b2df06.e5d7190a.14a05.0094@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90363:92b4fb5b9e58
Date: 2017-02-26 14:57 +0100
http://bitbucket.org/pypy/pypy/changeset/92b4fb5b9e58/

Log:	Fix issue 2289 (hopefully)

diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -280,8 +280,9 @@
                     # If L, V -> LV
                     current = SBase + ((current - LBase)*VCount + (next - VBase)) * TCount
                     continue
+                # Note: if next == TBase, leave LV unchanged
                 if (SBase <= current < SBase + SCount and
-                    TBase <= next < TBase + TCount and
+                    TBase < next < TBase + TCount and
                     (current - SBase) % TCount == 0):
                     # If LV, T -> LVT
                     current = current + (next - TBase)
diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py
--- a/pypy/module/unicodedata/test/test_hyp.py
+++ b/pypy/module/unicodedata/test/test_hyp.py
@@ -1,6 +1,6 @@
 import pytest
 try:
-    from hypothesis import given, strategies as st, example
+    from hypothesis import given, strategies as st, example, settings
 except ImportError:
     pytest.skip("hypothesis required")
 
@@ -40,6 +40,7 @@
 
 @pytest.mark.parametrize('NF1, NF2, NF3', compositions)
 @example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at settings(max_examples=1000)
 @given(s=st.text())
 def test_composition(s, space, NF1, NF2, NF3):
     norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]]

From pypy.commits at gmail.com  Sun Feb 26 08:58:28 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 26 Feb 2017 05:58:28 -0800 (PST)
Subject: [pypy-commit] pypy default: Interp-level version on
 extra_test/test_unicode.py
Message-ID: <58b2df04.1a0e2e0a.4eca.a749@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90362:bba953a1075b
Date: 2017-02-26 10:35 +0100
http://bitbucket.org/pypy/pypy/changeset/bba953a1075b/

Log:	Interp-level version on extra_test/test_unicode.py

diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/unicodedata/test/test_hyp.py
@@ -0,0 +1,46 @@
+import pytest
+try:
+    from hypothesis import given, strategies as st, example
+except ImportError:
+    pytest.skip("hypothesis required")
+
+from pypy.module.unicodedata.interp_ucd import ucd
+
+def make_normalization(space, NF_code):
+    def normalize(s):
+        w_s = space.newunicode(s)
+        w_res = ucd.normalize(space, NF_code, w_s)
+        return space.unwrap(w_res)
+    return normalize
+
+all_forms = ['NFC', 'NFD', 'NFKC', 'NFKD']
+
+# For every (n1, n2, n3) triple, applying n1 then n2 must be the same
+# as applying n3.
+# Reference: http://unicode.org/reports/tr15/#Design_Goals
+compositions = [
+    ('NFC', 'NFC', 'NFC'),
+    ('NFC', 'NFD', 'NFD'),
+    ('NFC', 'NFKC', 'NFKC'),
+    ('NFC', 'NFKD', 'NFKD'),
+    ('NFD', 'NFC', 'NFC'),
+    ('NFD', 'NFD', 'NFD'),
+    ('NFD', 'NFKC', 'NFKC'),
+    ('NFD', 'NFKD', 'NFKD'),
+    ('NFKC', 'NFC', 'NFKC'),
+    ('NFKC', 'NFD', 'NFKD'),
+    ('NFKC', 'NFKC', 'NFKC'),
+    ('NFKC', 'NFKD', 'NFKD'),
+    ('NFKD', 'NFC', 'NFKC'),
+    ('NFKD', 'NFD', 'NFKD'),
+    ('NFKD', 'NFKC', 'NFKC'),
+    ('NFKD', 'NFKD', 'NFKD'),
+]
+
+
+ at pytest.mark.parametrize('NF1, NF2, NF3', compositions)
+ at example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at given(s=st.text())
+def test_composition(s, space, NF1, NF2, NF3):
+    norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]]
+    assert norm2(norm1(s)) == norm3(s)

From pypy.commits at gmail.com  Sun Feb 26 08:58:33 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 26 Feb 2017 05:58:33 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58b2df09.e5d7190a.14a05.0096@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r90364:e28dd1841ff7
Date: 2017-02-26 14:57 +0100
http://bitbucket.org/pypy/pypy/changeset/e28dd1841ff7/

Log:	merge heads

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -156,19 +156,30 @@
 Does PyPy have a GIL?  Why?
 -------------------------------------------------
 
-Yes, PyPy has a GIL.  Removing the GIL is very hard.  The problems are
-essentially the same as with CPython (including the fact that our
-garbage collectors are not thread-safe so far).  Fixing it is possible,
-as shown by Jython and IronPython, but difficult.  It would require
-adapting the whole source code of PyPy, including subtle decisions about
-whether some effects are ok or not for the user (i.e. the Python
-programmer).
+Yes, PyPy has a GIL.  Removing the GIL is very hard.  On top of CPython,
+you have two problems:  (1) GC, in this case reference counting; (2) the
+whole Python language.
 
-Instead, since 2012, there is work going on on a still very experimental
-:doc:`Software Transactional Memory <stm>` (STM) version of PyPy.  This should give
-an alternative PyPy which works without a GIL, while at the same time
-continuing to give the Python programmer the complete illusion of having
-one.
+For PyPy, the hard issue is (2): by that I mean issues like what occurs
+if a mutable object is changed from one thread and read from another
+concurrently.  This is a problem for *any* mutable type: it needs
+careful review and fixes (fine-grained locks, mostly) through the
+*whole* Python interpreter.  It is a major effort, although not
+completely impossible, as Jython/IronPython showed.  This includes
+subtle decisions about whether some effects are ok or not for the user
+(i.e. the Python programmer).
+
+CPython has additionally the problem (1) of reference counting.  With
+PyPy, this sub-problem is simpler: we need to make our GC
+multithread-aware.  This is easier to do efficiently in PyPy than in
+CPython.  It doesn't solve the issue (2), though.
+
+Note that since 2012 there is work going on on a still very experimental
+:doc:`Software Transactional Memory <stm>` (STM) version of PyPy.  This
+should give an alternative PyPy which works without a GIL, while at the
+same time continuing to give the Python programmer the complete illusion
+of having one.  This work is currently a bit stalled because of its own
+technical difficulties.
 
 
 Is PyPy more clever than CPython about Tail Calls?
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -143,6 +143,13 @@
 behavior is also available on Python 2.x or for the ``dict`` type by
 calling ``__pypy__.move_to_end(dict, key, last=True)``.
 
+
+.. branch optinfo-into-bridges-3
+
+Improve the optimization of branchy Python code by retaining more information
+across failing guards.
+
+
 .. branch: space-newtext
 
 Internal refactoring of ``space.wrap()``, which is now replaced with
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -146,14 +146,13 @@
     def get_register_index_f(self, index):
         return ord(self.live_f[index])
 
-    def enumerate_vars(self, callback_i, callback_r, callback_f, spec, index):
+    def enumerate_vars(self, callback_i, callback_r, callback_f, spec):
         for i in range(self.get_register_count_i()):
-            index = callback_i(index, self.get_register_index_i(i))
+            callback_i(self.get_register_index_i(i))
         for i in range(self.get_register_count_r()):
-            index = callback_r(index, self.get_register_index_r(i))
+            callback_r(self.get_register_index_r(i))
         for i in range(self.get_register_count_f()):
-            index = callback_f(index, self.get_register_index_f(i))
-        return index
+            callback_f(self.get_register_index_f(i))
     enumerate_vars._annspecialcase_ = 'specialize:arg(4)'
 
 _liveness_cache = {}
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -85,13 +85,14 @@
     """ This represents ops() with a jump at the end that goes to some
     loop, we need to deal with virtual state and inlining of short preamble
     """
-    def __init__(self, trace, runtime_boxes, call_pure_results=None,
+    def __init__(self, trace, runtime_boxes, resumestorage=None, call_pure_results=None,
                  enable_opts=None, inline_short_preamble=False):
         self.trace = trace
         self.runtime_boxes = runtime_boxes
         self.call_pure_results = call_pure_results
         self.enable_opts = enable_opts
         self.inline_short_preamble = inline_short_preamble
+        self.resumestorage = resumestorage
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -100,7 +101,8 @@
         return opt.optimize_bridge(self.trace, self.runtime_boxes,
                                    self.call_pure_results,
                                    self.inline_short_preamble,
-                                   self.box_names_memo)
+                                   self.box_names_memo,
+                                   self.resumestorage)
 
 class UnrolledLoopData(CompileData):
     """ This represents label() ops jump with extra info that's from the
@@ -870,10 +872,9 @@
 
 
 class ResumeGuardDescr(AbstractResumeGuardDescr):
-    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+    _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals',
                'rd_pendingfields', 'status')
     rd_numb = lltype.nullptr(NUMBERING)
-    rd_count = 0
     rd_consts = None
     rd_virtuals = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
@@ -882,7 +883,6 @@
         if isinstance(other, ResumeGuardCopiedDescr):
             other = other.prev
         assert isinstance(other, ResumeGuardDescr)
-        self.rd_count = other.rd_count
         self.rd_consts = other.rd_consts
         self.rd_pendingfields = other.rd_pendingfields
         self.rd_virtuals = other.rd_virtuals
@@ -895,7 +895,6 @@
 
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
-        self.rd_count = len(boxes)
         self.store_hash(metainterp_sd)
 
     def clone(self):
@@ -1077,7 +1076,15 @@
     call_pure_results = metainterp.call_pure_results
 
     if metainterp.history.ends_with_jump:
-        data = BridgeCompileData(trace, runtime_boxes,
+        if isinstance(resumekey, ResumeGuardCopiedDescr):
+            key = resumekey.prev
+            assert isinstance(key, ResumeGuardDescr)
+        elif isinstance(resumekey, ResumeFromInterpDescr):
+            key = None
+        else:
+            key = resumekey
+            assert isinstance(key, ResumeGuardDescr)
+        data = BridgeCompileData(trace, runtime_boxes, key,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts,
                                  inline_short_preamble=inline_short_preamble)
diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py
--- a/rpython/jit/metainterp/opencoder.py
+++ b/rpython/jit/metainterp/opencoder.py
@@ -66,7 +66,7 @@
         assert isinstance(snapshot, TopSnapshot)
         self.vable_array = snapshot.vable_array
         self.vref_array = snapshot.vref_array
-        self.size = len(self.vable_array) + len(self.vref_array) + 2
+        self.size = len(self.vable_array) + len(self.vref_array) + 3
         jc_index, pc = unpack_uint(snapshot.packed_jitcode_pc)
         self.framestack = []
         if jc_index == 2**16-1:
diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -0,0 +1,135 @@
+""" Code to feed information from the optimizer via the resume code into the
+optimizer of the bridge attached to a guard. """
+
+from rpython.jit.metainterp import resumecode
+
+
+# adds the following sections at the end of the resume code:
+#
+# ---- known classes
+# <bitfield> size is the number of reference boxes in the liveboxes
+#            1 klass known
+#            0 klass unknown
+#            (the class is found by actually looking at the runtime value)
+#            the bits are bunched in bunches of 7
+#
+# ---- heap knowledge
+# <length>
+# (<box1> <descr> <box2>) length times, if getfield(box1, descr) == box2
+#                         both boxes should be in the liveboxes
+#
+# ----
+
+
+# maybe should be delegated to the optimization classes?
+
+def tag_box(box, liveboxes_from_env, memo):
+    from rpython.jit.metainterp.history import Const
+    if isinstance(box, Const):
+        return memo.getconst(box)
+    else:
+        return liveboxes_from_env[box] # has to exist
+
+def decode_box(resumestorage, tagged, liveboxes, cpu):
+    from rpython.jit.metainterp.resume import untag, TAGCONST, TAGINT, TAGBOX
+    from rpython.jit.metainterp.resume import NULLREF, TAG_CONST_OFFSET, tagged_eq
+    from rpython.jit.metainterp.history import ConstInt
+    num, tag = untag(tagged)
+    # NB: the TAGVIRTUAL case can't happen here, because this code runs after
+    # virtuals are already forced again
+    if tag == TAGCONST:
+        if tagged_eq(tagged, NULLREF):
+            box = cpu.ts.CONST_NULL
+        else:
+            box = resumestorage.rd_consts[num - TAG_CONST_OFFSET]
+    elif tag == TAGINT:
+        box = ConstInt(num)
+    elif tag == TAGBOX:
+        box = liveboxes[num]
+    else:
+        raise AssertionError("unreachable")
+    return box
+
+def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo):
+    available_boxes = {}
+    for box in liveboxes:
+        if box is not None and box in liveboxes_from_env:
+            available_boxes[box] = None
+    metainterp_sd = optimizer.metainterp_sd
+
+    # class knowledge is stored as bits, true meaning the class is known, false
+    # means unknown. on deserializing we look at the bits, and read the runtime
+    # class for the known classes (which has to be the same in the bridge) and
+    # mark that as known. this works for guard_class too: the class is only
+    # known *after* the guard
+    bitfield = 0
+    shifts = 0
+    for box in liveboxes:
+        if box is None or box.type != "r":
+            continue
+        info = optimizer.getptrinfo(box)
+        known_class = info is not None and info.get_known_class(optimizer.cpu) is not None
+        bitfield <<= 1
+        bitfield |= known_class
+        shifts += 1
+        if shifts == 6:
+            numb_state.append_int(bitfield)
+            bitfield = shifts = 0
+    if shifts:
+        numb_state.append_int(bitfield << (6 - shifts))
+
+    # heap knowledge: we store triples of known heap fields in non-virtual
+    # structs
+    # XXX could be extended to arrays
+    if optimizer.optheap:
+        triples = optimizer.optheap.serialize_optheap(available_boxes)
+        # can only encode descrs that have a known index into
+        # metainterp_sd.all_descrs
+        triples = [triple for triple in triples if triple[1].descr_index != -1]
+        numb_state.append_int(len(triples))
+        for box1, descr, box2 in triples:
+            index = descr.descr_index
+            numb_state.append_short(tag_box(box1, liveboxes_from_env, memo))
+            numb_state.append_int(index)
+            numb_state.append_short(tag_box(box2, liveboxes_from_env, memo))
+    else:
+        numb_state.append_int(0)
+
+def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes):
+    reader = resumecode.Reader(resumestorage.rd_numb)
+    assert len(frontend_boxes) == len(liveboxes)
+    metainterp_sd = optimizer.metainterp_sd
+
+    # skip resume section
+    startcount = reader.next_item()
+    reader.jump(startcount - 1)
+
+    # class knowledge
+    bitfield = 0
+    mask = 0
+    for i, box in enumerate(liveboxes):
+        if box.type != "r":
+            continue
+        if not mask:
+            bitfield = reader.next_item()
+            mask = 0b100000
+        class_known = bitfield & mask
+        mask >>= 1
+        if class_known:
+            cls = optimizer.cpu.ts.cls_of_box(frontend_boxes[i])
+            optimizer.make_constant_class(box, cls)
+
+    # heap knowledge
+    if not optimizer.optheap:
+        return
+    length = reader.next_item()
+    result = []
+    for i in range(length):
+        tagged = reader.next_item()
+        box1 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
+        index = reader.next_item()
+        descr = metainterp_sd.all_descrs[index]
+        tagged = reader.next_item()
+        box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
+        result.append((box1, descr, box2))
+    optimizer.optheap.deserialize_optheap(result)
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -48,6 +48,7 @@
         # that has a non-None entry at
         # info._fields[descr.get_index()]
         # must be in cache_infos
+        assert structop.type == 'r'
         self.cached_structs.append(structop)
         self.cached_infos.append(info)
 
@@ -693,6 +694,36 @@
         self._seen_guard_not_invalidated = True
         return self.emit(op)
 
+    def serialize_optheap(self, available_boxes):
+        result = []
+        for descr, cf in self.cached_fields.iteritems():
+            if cf._lazy_set:
+                continue # XXX safe default for now
+            parent_descr = descr.get_parent_descr()
+            if not parent_descr.is_object():
+                continue # XXX could be extended to non-instance objects
+            for i, box1 in enumerate(cf.cached_structs):
+                if box1 not in available_boxes:
+                    continue
+                structinfo = cf.cached_infos[i]
+                box2 = structinfo.getfield(descr).get_box_replacement()
+                if isinstance(box2, Const) or box2 in available_boxes:
+                    result.append((box1, descr, box2))
+        return result
+
+    def deserialize_optheap(self, triples):
+        for box1, descr, box2 in triples:
+            parent_descr = descr.get_parent_descr()
+            assert parent_descr.is_object()
+            structinfo = box1.get_forwarded()
+            if not isinstance(structinfo, info.AbstractVirtualPtrInfo):
+                structinfo = info.InstancePtrInfo(parent_descr)
+                structinfo.init_fields(parent_descr, descr.get_index())
+                box1.set_forwarded(structinfo)
+
+            cf = self.field_cache(descr)
+            structinfo.setfield(descr, box1, box2, optheap=self, cf=cf)
+
 
 dispatch_opt = make_dispatcher_method(OptHeap, 'optimize_',
                                       default=OptHeap.emit)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -70,7 +70,7 @@
     def same_info(self, other):
         return self is other
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
         return None
 
     def getstrhash(self, op, mode):
@@ -777,21 +777,20 @@
             # XXX we can do better if we know it's an array
             return IntLowerBound(0)
         else:
-            return ConstIntBound(self.getstrlen(None, None, mode).getint())
-    
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+            return ConstIntBound(self.getstrlen1(mode))
+
+    def getstrlen(self, op, string_optimizer, mode):
+        return ConstInt(self.getstrlen1(mode))
+
+    def getstrlen1(self, mode):
         from rpython.jit.metainterp.optimizeopt import vstring
         
         if mode is vstring.mode_string:
             s = self._unpack_str(vstring.mode_string)
-            if s is None:
-                return None
-            return ConstInt(len(s))
+            return len(s)
         else:
             s = self._unpack_str(vstring.mode_unicode)            
-            if s is None:
-                return None
-            return ConstInt(len(s))
+            return len(s)
 
     def getstrhash(self, op, mode):
         from rpython.jit.metainterp.optimizeopt import vstring
@@ -812,7 +811,7 @@
         from rpython.jit.metainterp.optimizeopt import vstring
         from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
 
-        lgt = self.getstrlen(op, string_optimizer, mode, False)
+        lgt = self.getstrlen(op, string_optimizer, mode)
         return vstring.copy_str_content(string_optimizer, self._const,
                                         targetbox, CONST_0, offsetbox,
                                         lgt, mode)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -297,6 +297,7 @@
         self.optrewrite = None
         self.optearlyforce = None
         self.optunroll = None
+        self._really_emitted_operation = None
 
         self._last_guard_op = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -30,6 +30,7 @@
         self.add_guard_future_condition(bridge)
         trace = oparser.convert_loop_to_trace(bridge, FakeMetaInterpStaticData(self.cpu))
         data = compile.BridgeCompileData(trace, self.convert_values(bridge.operations[-1].getarglist(), bridge_values),
+                                         None,
                                          enable_opts=self.enable_opts,
                             inline_short_preamble=inline_short_preamble)
         bridge_info, ops = self._do_optimize_loop(data)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -232,9 +232,15 @@
         return label_vs
 
     def optimize_bridge(self, trace, runtime_boxes, call_pure_results,
-                        inline_short_preamble, box_names_memo):
+                        inline_short_preamble, box_names_memo, resumestorage):
+        from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+        frontend_inputargs = trace.inputargs
         trace = trace.get_iter()
         self._check_no_forwarding([trace.inputargs])
+        if resumestorage:
+            deserialize_optimizer_knowledge(self.optimizer,
+                                            resumestorage, frontend_inputargs,
+                                            trace.inputargs)
         info, ops = self.optimizer.propagate_all_forward(trace,
             call_pure_results, False)
         jump_op = info.jump_op
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -752,7 +752,6 @@
 
     def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)):
         self.setref_base(r)
-        self.datatype = 'r'
 
     def reset_value(self):
         self.setref_base(lltype.nullptr(llmemory.GCREF.TO))
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -162,25 +162,13 @@
 UNINITIALIZED = tag(-2, TAGCONST)   # used for uninitialized string characters
 TAG_CONST_OFFSET = 0
 
-class NumberingState(object):
+class NumberingState(resumecode.Writer):
     def __init__(self, size):
+        resumecode.Writer.__init__(self, size)
         self.liveboxes = {}
-        self.current = [rffi.cast(rffi.SHORT, 0)] * size
-        self._pos = 0
         self.num_boxes = 0
         self.num_virtuals = 0
 
-    def append_short(self, item):
-        self.current[self._pos] = item
-        self._pos += 1
-
-    def append_int(self, item):
-        short = rffi.cast(rffi.SHORT, item)
-        assert rffi.cast(lltype.Signed, short) == item
-        return self.append_short(short)
-
-    def create_numbering(self):
-        return resumecode.create_numbering(self.current)
 
 class ResumeDataLoopMemo(object):
 
@@ -268,6 +256,8 @@
     def number(self, optimizer, position, trace):
         snapshot_iter = trace.get_snapshot_iter(position)
         numb_state = NumberingState(snapshot_iter.size)
+        numb_state.append_int(0) # patch later: size of resume section
+        numb_state.append_int(0) # patch later: number of failargs
 
         arr = snapshot_iter.vable_array
 
@@ -287,6 +277,7 @@
             numb_state.append_int(pc)
             self._number_boxes(
                     snapshot_iter, snapshot.box_array, optimizer, numb_state)
+        numb_state.patch_current_size(0)
 
         return numb_state
 
@@ -471,6 +462,9 @@
         self._number_virtuals(liveboxes, optimizer, num_virtuals)
         self._add_pending_fields(optimizer, pending_setfields)
 
+        numb_state.patch(1, len(liveboxes))
+
+        self._add_optimizer_sections(numb_state, liveboxes, liveboxes_from_env)
         storage.rd_numb = numb_state.create_numbering()
         storage.rd_consts = self.memo.consts
         return liveboxes[:]
@@ -590,6 +584,12 @@
                 return self.liveboxes_from_env[box]
             return self.liveboxes[box]
 
+    def _add_optimizer_sections(self, numb_state, liveboxes, liveboxes_from_env):
+        # add extra information about things the optimizer learned
+        from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge
+        serialize_optimizer_knowledge(
+            self.optimizer, numb_state, liveboxes, liveboxes_from_env, self.memo)
+
 class AbstractVirtualInfo(object):
     kind = REF
     is_about_raw = False
@@ -931,9 +931,10 @@
 
     def _init(self, cpu, storage):
         self.cpu = cpu
-        self.numb = storage.rd_numb
-        self.cur_index = 0
-        self.count = storage.rd_count
+        self.resumecodereader = resumecode.Reader(storage.rd_numb)
+        items_resume_section = self.resumecodereader.next_item()
+        self.items_resume_section = items_resume_section
+        self.count = self.resumecodereader.next_item()
         self.consts = storage.rd_consts
 
     def _prepare(self, storage):
@@ -941,14 +942,21 @@
         self._prepare_pendingfields(storage.rd_pendingfields)
 
     def read_jitcode_pos_pc(self):
-        jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb,
-            self.cur_index)
-        pc, self.cur_index = resumecode.numb_next_item(self.numb,
-            self.cur_index)
+        jitcode_pos = self.resumecodereader.next_item()
+        pc = self.resumecodereader.next_item()
         return jitcode_pos, pc
 
+    def next_int(self):
+        return self.decode_int(self.resumecodereader.next_item())
+
+    def next_ref(self):
+        return self.decode_ref(self.resumecodereader.next_item())
+
+    def next_float(self):
+        return self.decode_float(self.resumecodereader.next_item())
+
     def done_reading(self):
-        return self.cur_index >= len(self.numb.code)
+        return self.resumecodereader.items_read >= self.items_resume_section
 
     def getvirtual_ptr(self, index):
         # Returns the index'th virtual, building it lazily if needed.
@@ -1025,29 +1033,22 @@
     def _prepare_next_section(self, info):
         # Use info.enumerate_vars(), normally dispatching to
         # rpython.jit.codewriter.jitcode.  Some tests give a different 'info'.
-        self.cur_index = info.enumerate_vars(self._callback_i,
-                                        self._callback_r,
-                                        self._callback_f,
-                                        self.unique_id,  # <-- annotation hack
-                                        self.cur_index)
+        info.enumerate_vars(self._callback_i,
+                            self._callback_r,
+                            self._callback_f,
+                            self.unique_id)  # <-- annotation hack
 
-    def _callback_i(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_int(item)
+    def _callback_i(self, register_index):
+        value = self.next_int()
         self.write_an_int(register_index, value)
-        return index
 
-    def _callback_r(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_ref(item)
+    def _callback_r(self, register_index):
+        value = self.next_ref()
         self.write_a_ref(register_index, value)
-        return index
 
-    def _callback_f(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_float(item)
+    def _callback_f(self, register_index):
+        value = self.next_float()
         self.write_a_float(register_index, value)
-        return index
 
 # ---------- when resuming for pyjitpl.py, make boxes ----------
 
@@ -1057,6 +1058,7 @@
     boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info,
                                                       greenfield_info)
     virtualizable_boxes, virtualref_boxes = boxes
+
     while not resumereader.done_reading():
         jitcode_pos, pc = resumereader.read_jitcode_pos_pc()
         jitcode = metainterp.staticdata.jitcodes[jitcode_pos]
@@ -1076,7 +1078,7 @@
         self._init(metainterp.cpu, storage)
         self.deadframe = deadframe
         self.metainterp = metainterp
-        self.liveboxes = [None] * storage.rd_count
+        self.liveboxes = [None] * self.count
         self._prepare(storage)
 
     def consume_boxes(self, info, boxes_i, boxes_r, boxes_f):
@@ -1085,42 +1087,30 @@
         self.boxes_f = boxes_f
         self._prepare_next_section(info)
 
-    def consume_virtualizable_boxes(self, vinfo, index):
+    def consume_virtualizable_boxes(self, vinfo):
         # we have to ignore the initial part of 'nums' (containing vrefs),
         # find the virtualizable from nums[-1], and use it to know how many
         # boxes of which type we have to return.  This does not write
         # anything into the virtualizable.
-        numb = self.numb
-        item, index = resumecode.numb_next_item(numb, index)
-        virtualizablebox = self.decode_ref(item)
+        virtualizablebox = self.next_ref()
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox)
-        return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox,
-            numb, index)
+        return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox)
 
-    def consume_virtualref_boxes(self, index):
+    def consume_virtualref_boxes(self):
         # Returns a list of boxes, assumed to be all BoxPtrs.
         # We leave up to the caller to call vrefinfo.continue_tracing().
-        size, index = resumecode.numb_next_item(self.numb, index)
-        if size == 0:
-            return [], index
-        lst = []
-        for i in range(size * 2):
-            item, index = resumecode.numb_next_item(self.numb, index)
-            lst.append(self.decode_ref(item))
-        return lst, index
+        size = self.resumecodereader.next_item()
+        return [self.next_ref() for i in range(size * 2)]
 
     def consume_vref_and_vable_boxes(self, vinfo, ginfo):
-        vable_size, index = resumecode.numb_next_item(self.numb, 0)
+        vable_size = self.resumecodereader.next_item()
         if vinfo is not None:
-            virtualizable_boxes, index = self.consume_virtualizable_boxes(vinfo,
-                                                                          index)
+            virtualizable_boxes = self.consume_virtualizable_boxes(vinfo)
         elif ginfo is not None:
-            item, index = resumecode.numb_next_item(self.numb, index)
-            virtualizable_boxes = [self.decode_ref(item)]
+            virtualizable_boxes = [self.next_ref()]
         else:
             virtualizable_boxes = None
-        virtualref_boxes, index = self.consume_virtualref_boxes(index)
-        self.cur_index = index
+        virtualref_boxes = self.consume_virtualref_boxes()
         return virtualizable_boxes, virtualref_boxes
 
     def allocate_with_vtable(self, descr=None):
@@ -1297,7 +1287,7 @@
         self.liveboxes[num] = box
         return box
 
-    def decode_box_of_type(self, TYPE, tagged):
+    def next_box_of_type(self, TYPE):
         kind = getkind(TYPE)
         if kind == 'int':
             kind = INT
@@ -1307,8 +1297,8 @@
             kind = FLOAT
         else:
             raise AssertionError(kind)
-        return self.decode_box(tagged, kind)
-    decode_box_of_type._annspecialcase_ = 'specialize:arg(1)'
+        return self.decode_box(self.resumecodereader.next_item(), kind)
+    next_box_of_type._annspecialcase_ = 'specialize:arg(1)'
 
     def write_an_int(self, index, box):
         self.boxes_i[index] = box
@@ -1397,64 +1387,54 @@
         info = blackholeinterp.get_current_position_info()
         self._prepare_next_section(info)
 
-    def consume_virtualref_info(self, vrefinfo, index):
+    def consume_virtualref_info(self, vrefinfo):
         # we have to decode a list of references containing pairs
         # [..., virtual, vref, ...] and returns the index at the end
-        size, index = resumecode.numb_next_item(self.numb, index)
+        size = self.resumecodereader.next_item()
         if vrefinfo is None or size == 0:
             assert size == 0
-            return index
+            return
         for i in range(size):
-            virtual_item, index = resumecode.numb_next_item(
-                self.numb, index)
-            vref_item, index = resumecode.numb_next_item(
-                self.numb, index)
-            virtual = self.decode_ref(virtual_item)
-            vref = self.decode_ref(vref_item)
+            virtual = self.next_ref()
+            vref = self.next_ref()
             # For each pair, we store the virtual inside the vref.
             vrefinfo.continue_tracing(vref, virtual)
-        return index
 
-    def consume_vable_info(self, vinfo, index):
+    def consume_vable_info(self, vinfo):
         # we have to ignore the initial part of 'nums' (containing vrefs),
         # find the virtualizable from nums[-1], load all other values
         # from the CPU stack, and copy them into the virtualizable
-        numb = self.numb
-        item, index = resumecode.numb_next_item(self.numb, index)
-        virtualizable = self.decode_ref(item)
+        virtualizable = self.next_ref()
         # just reset the token, we'll force it later
         vinfo.reset_token_gcref(virtualizable)
-        index = vinfo.write_from_resume_data_partial(virtualizable, self,
-            index, numb)
-        return index
+        vinfo.write_from_resume_data_partial(virtualizable, self)
 
-    def load_value_of_type(self, TYPE, tagged):
+    def load_next_value_of_type(self, TYPE):
         from rpython.jit.metainterp.warmstate import specialize_value
         kind = getkind(TYPE)
         if kind == 'int':
-            x = self.decode_int(tagged)
+            x = self.next_int()
         elif kind == 'ref':
-            x = self.decode_ref(tagged)
+            x = self.next_ref()
         elif kind == 'float':
-            x = self.decode_float(tagged)
+            x = self.next_float()
         else:
             raise AssertionError(kind)
         return specialize_value(TYPE, x)
-    load_value_of_type._annspecialcase_ = 'specialize:arg(1)'
+    load_next_value_of_type._annspecialcase_ = 'specialize:arg(1)'
 
     def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo):
-        vable_size, index = resumecode.numb_next_item(self.numb, 0)
+        vable_size = self.resumecodereader.next_item()
         if self.resume_after_guard_not_forced != 2:
             if vinfo is not None:
-                index = self.consume_vable_info(vinfo, index)
+                self.consume_vable_info(vinfo)
             if ginfo is not None:
-                _, index = resumecode.numb_next_item(self.numb, index)
-            index = self.consume_virtualref_info(vrefinfo, index)
+                _ = self.resumecodereader.next_item()
+            self.consume_virtualref_info(vrefinfo)
         else:
-            index = resumecode.numb_next_n_items(self.numb, vable_size, index)
-            vref_size, index = resumecode.numb_next_item(self.numb, index)
-            index = resumecode.numb_next_n_items(self.numb, vref_size * 2, index)
-        self.cur_index = index 
+            self.resumecodereader.jump(vable_size)
+            vref_size = self.resumecodereader.next_item()
+            self.resumecodereader.jump(vref_size * 2)
 
     def allocate_with_vtable(self, descr=None):
         from rpython.jit.metainterp.executor import exec_new_with_vtable
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -1,6 +1,9 @@
 
 """ Resume bytecode. It goes as following:
 
+  # ----- resume section
+  [total size of resume section]
+  [number of failargs]
   [<length> <virtualizable object> <numb> <numb> <numb>]    if vinfo is not None
    -OR-
   [1 <ginfo object>]                                        if ginfo is not None
@@ -13,10 +16,14 @@
   [<pc> <jitcode> <numb> <numb>]
   ...
 
-  until the length of the array.
+  until the size of the resume section
+
+  # ----- optimization section
+  <more code>                                      further sections according to bridgeopt.py
 """
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib import objectmodel
 
 NUMBERINGP = lltype.Ptr(lltype.GcForwardReference())
 NUMBERING = lltype.GcStruct('Numbering',
@@ -24,33 +31,24 @@
 NUMBERINGP.TO.become(NUMBERING)
 NULL_NUMBER = lltype.nullptr(NUMBERING)
 
-def create_numbering(lst, total=-1):
-    if total == -1:
-        total = len(lst)
-    result = []
-    for i in range(total):
-        item = lst[i]
-        item = rffi.cast(lltype.Signed, item)
-        item *= 2
-        if item < 0:
-            item = -1 - item
+def append_numbering(lst, item):
+    item = rffi.cast(lltype.Signed, item)
+    item *= 2
+    if item < 0:
+        item = -1 - item
 
-        assert item >= 0
-        if item < 2**7:
-            result.append(rffi.cast(rffi.UCHAR, item))
-        elif item < 2**14:
-            result.append(rffi.cast(rffi.UCHAR, item | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, item >> 7))
-        else:
-            assert item < 2**16
-            result.append(rffi.cast(rffi.UCHAR, item | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, item >> 14))
+    assert item >= 0
+    if item < 2**7:
+        lst.append(rffi.cast(rffi.UCHAR, item))
+    elif item < 2**14:
+        lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, item >> 7))
+    else:
+        assert item < 2**16
+        lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, item >> 14))
 
-    numb = lltype.malloc(NUMBERING, len(result))
-    for i in range(len(result)):
-        numb.code[i] = result[i]
-    return numb
 
 def numb_next_item(numb, index):
     value = rffi.cast(lltype.Signed, numb.code[index])
@@ -81,3 +79,64 @@
         next, i = numb_next_item(numb, i)
         l.append(next)
     return l
+
+class Writer(object):
+    def __init__(self, size=0):
+        self.current = objectmodel.newlist_hint(size)
+
+    def append_short(self, item):
+        self.current.append(item)
+
+    def append_int(self, item):
+        short = rffi.cast(rffi.SHORT, item)
+        assert rffi.cast(lltype.Signed, short) == item
+        return self.append_short(short)
+
+    def create_numbering(self):
+        final = objectmodel.newlist_hint(len(self.current) * 3)
+        for item in self.current:
+            append_numbering(final, item)
+        numb = lltype.malloc(NUMBERING, len(final))
+        for i, elt in enumerate(final):
+            numb.code[i] = elt
+        return numb
+
+    def patch_current_size(self, index):
+        self.patch(index, len(self.current))
+
+    def patch(self, index, item):
+        self.current[index] = item
+
+def create_numbering(l):
+    w = Writer()
+    for item in l:
+        w.append_int(item)
+    return w.create_numbering()
+
+
+class Reader(object):
+    def __init__(self, code):
+        self.code = code
+        self.cur_pos = 0 # index into the code
+        self.items_read = 0 # number of items read
+
+    def next_item(self):
+        result, self.cur_pos = numb_next_item(self.code, self.cur_pos)
+        self.items_read += 1
+        return result
+
+    def peek(self):
+        result, _ = numb_next_item(self.code, self.cur_pos)
+        return result
+
+    def jump(self, size):
+        """ jump n items forward without returning anything """
+        index = self.cur_pos
+        for i in range(size):
+            _, index = numb_next_item(self.code, index)
+        self.items_read += size
+        self.cur_pos = index
+
+    def unpack(self):
+        # mainly for debugging
+        return unpack_numbering(self.code)
diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_bridgeopt.py
@@ -0,0 +1,188 @@
+# tests that check that information is fed from the optimizer into the bridges
+
+import math
+from rpython.rlib import jit
+from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge
+from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+from rpython.jit.metainterp.resoperation import InputArgRef, InputArgInt
+from rpython.jit.metainterp.resume import NumberingState
+from rpython.jit.metainterp.resumecode import unpack_numbering
+from rpython.jit.metainterp.optimizeopt.info import InstancePtrInfo
+
+from hypothesis import strategies, given
+
+class FakeTS(object):
+    def __init__(self, dct):
+        self.dct = dct
+
+    def cls_of_box(self, box):
+        return self.dct[box]
+
+
+class FakeCPU(object):
+    def __init__(self, dct):
+        self.ts = FakeTS(dct)
+
+class FakeOptimizer(object):
+    metainterp_sd = None
+    optheap = None
+
+    def __init__(self, dct={}, cpu=None):
+        self.dct = dct
+        self.constant_classes = {}
+        self.cpu = cpu
+
+    def getptrinfo(self, arg):
+        return self.dct.get(arg, None)
+
+    def make_constant_class(self, arg, cls):
+        self.constant_classes[arg] = cls
+
+class FakeClass(object):
+    pass
+
+class FakeStorage(object):
+    def __init__(self, numb):
+        self.rd_numb = numb
+
+def test_known_classes():
+    box1 = InputArgRef()
+    box2 = InputArgRef()
+    box3 = InputArgRef()
+
+    cls = FakeClass()
+    dct = {box1: InstancePtrInfo(known_class=cls)}
+    optimizer = FakeOptimizer(dct)
+
+    numb_state = NumberingState(4)
+    numb_state.append_int(1) # size of resume block
+    liveboxes = [InputArgInt(), box2, box1, box3]
+
+    serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
+
+    assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0]
+
+    rbox1 = InputArgRef()
+    rbox2 = InputArgRef()
+    rbox3 = InputArgRef()
+    after_optimizer = FakeOptimizer(cpu=FakeCPU({rbox1: cls}))
+    deserialize_optimizer_knowledge(
+        after_optimizer, FakeStorage(numb_state.create_numbering()),
+        [InputArgInt(), rbox2, rbox1, rbox3], liveboxes)
+    assert box1 in after_optimizer.constant_classes
+    assert box2 not in after_optimizer.constant_classes
+    assert box3 not in after_optimizer.constant_classes
+
+
+box_strategy = strategies.builds(InputArgInt) | strategies.builds(InputArgRef)
+tuples = strategies.tuples(box_strategy, strategies.booleans()).filter(
+        lambda (box, known_class): isinstance(box, InputArgRef) or not known_class)
+boxes_known_classes = strategies.lists(tuples, min_size=1)
+
+ at given(boxes_known_classes)
+def test_random_class_knowledge(boxes_known_classes):
+    cls = FakeClass()
+    dct1 = {box: InstancePtrInfo(known_class=cls)
+              for box, known_class in boxes_known_classes
+                  if known_class}
+    optimizer = FakeOptimizer(dct1)
+
+    refboxes = [box for (box, _) in boxes_known_classes
+                    if isinstance(box, InputArgRef)]
+
+    numb_state = NumberingState(1)
+    numb_state.append_int(1) # size of resume block
+    liveboxes = [box for (box, _) in boxes_known_classes]
+
+    serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
+
+    assert len(numb_state.create_numbering().code) == 2 + math.ceil(len(refboxes) / 6.0)
+
+    dct = {box: cls
+              for box, known_class in boxes_known_classes
+                  if known_class}
+    after_optimizer = FakeOptimizer(cpu=FakeCPU(dct))
+    deserialize_optimizer_knowledge(
+        after_optimizer, FakeStorage(numb_state.create_numbering()),
+        liveboxes, liveboxes)
+    for box, known_class in boxes_known_classes:
+        assert (box in after_optimizer.constant_classes) == known_class
+
+class TestOptBridge(LLJitMixin):
+    # integration tests
+    def test_bridge_guard_class(self):
+        myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a'])
+        class A(object):
+            def f(self):
+                return 1
+        class B(A):
+            def f(self):
+                return 2
+        def f(x, y, n):
+            if x:
+                a = A()
+            else:
+                a = B()
+            a.x = 0
+            res = 0
+            while y > 0:
+                myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a)
+                res += a.f()
+                a.x += 1
+                if y > n:
+                    res += 1
+                res += a.f()
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 32, 16])
+        assert res == f(6, 32, 16)
+        self.check_trace_count(3)
+        self.check_resops(guard_class=1)
+
+    def test_bridge_field_read(self):
+        myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a'])
+        class A(object):
+            def f(self):
+                return 1
+        class B(A):
+            def f(self):
+                return 2
+        class M(object):
+            _immutable_fields_ = ['x']
+            def __init__(self, x):
+                self.x = x
+
+        m1 = M(1)
+        m2 = M(2)
+        def f(x, y, n):
+            if x:
+                a = A()
+                a.m = m1
+                a.n = n
+            else:
+                a = B()
+                a.m = m2
+                a.n = n
+            a.x = 0
+            res = 0
+            while y > 0:
+                myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a)
+                n1 = a.n
+                m = jit.promote(a.m)
+                res += m.x
+                a.x += 1
+                if y > n:
+                    res += 1
+                m = jit.promote(a.m)
+                res += m.x
+                res += n1 + a.n
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 32, 16])
+        assert res == f(6, 32, 16)
+        self.check_trace_count(3)
+        self.check_resops(guard_value=1)
+        self.check_resops(getfield_gc_i=4) # 3x a.x, 1x a.n
+        self.check_resops(getfield_gc_r=1) # in main loop
+
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -79,6 +79,7 @@
 def test_compile_loop():
     cpu = FakeCPU()
     staticdata = FakeMetaInterpStaticData()
+    staticdata.all_descrs = LLtypeMixin.cpu.setup_descrs()
     staticdata.cpu = cpu
     staticdata.jitlog = jl.JitLogger(cpu)
     staticdata.jitlog.trace_id = 1
diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py
--- a/rpython/jit/metainterp/test/test_greenfield.py
+++ b/rpython/jit/metainterp/test/test_greenfield.py
@@ -49,7 +49,7 @@
         #
         res = self.meta_interp(g, [7])
         assert res == -22
-        self.check_trace_count(6)
+        self.check_trace_count(4)
         self.check_resops(guard_value=0)
 
     def test_green_field_3(self):
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -36,10 +36,12 @@
     rd_consts = []
     rd_virtuals = None
     rd_pendingfields = None
-    rd_count = 0
 
 
 class FakeOptimizer(object):
+    metainterp_sd = None
+    optheap = None
+
     def __init__(self, trace=None):
         self.trace = trace
 
@@ -251,18 +253,17 @@
     def get_current_position_info(self):
         class MyInfo:
             @staticmethod
-            def enumerate_vars(callback_i, callback_r, callback_f, _, index):
+            def enumerate_vars(callback_i, callback_r, callback_f, _):
                 count_i = count_r = count_f = 0
                 for ARG in self.ARGS:
                     if ARG == lltype.Signed:
-                        index = callback_i(index, count_i); count_i += 1
+                        callback_i(count_i); count_i += 1
                     elif ARG == llmemory.GCREF:
-                        index = callback_r(index, count_r); count_r += 1
+                        callback_r(count_r); count_r += 1
                     elif ARG == longlong.FLOATSTORAGE:
-                        index = callback_f(index, count_f); count_f += 1
+                        callback_f(count_f); count_f += 1
                     else:
                         assert 0
-                return index
         return MyInfo()
 
     def setarg_i(self, index, value):
@@ -289,7 +290,8 @@
     assert bh.written_f == expected_f
 
 
-Numbering = create_numbering
+def Numbering(l):
+    return create_numbering([len(l)] + l) # prefix index to the end of thing
 
 def tagconst(i):
     return tag(i + TAG_CONST_OFFSET, TAGCONST)
@@ -299,12 +301,11 @@
     c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)]
     storage = Storage()
     storage.rd_consts = [c1, c2, c3]
-    numb = Numbering([tag(0, TAGBOX), tagconst(0),
+    numb = Numbering([3, tag(0, TAGBOX), tagconst(0),
                        NULLREF, tag(0, TAGBOX), tag(1, TAGBOX)] +
-                       [tagconst(1), tagconst(2)] + 
+                       [tagconst(1), tagconst(2)] +
                        [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)])
     storage.rd_numb = numb
-    storage.rd_count = 3
     #
     cpu = MyCPU([42, gcref1, -66])
     metainterp = MyMetaInterp(cpu)
@@ -345,7 +346,7 @@
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
-    numb = Numbering([tag(100, TAGINT)])
+    numb = Numbering([1, tag(100, TAGINT)])
     storage.rd_numb = numb
     #
     cpu = MyCPU([])
@@ -362,10 +363,9 @@
             return s
     class FakeStorage(object):
         rd_virtuals = [FakeVinfo(), None]
-        rd_numb = []
+        rd_numb = Numbering([1])
         rd_consts = []
         rd_pendingfields = None
-        rd_count = 0
     class FakeMetainterp(object):
         _already_allocated_resume_virtuals = None
         cpu = None
@@ -773,12 +773,12 @@
     assert untag(tagged) == (44, TAGINT)
     tagged = memo.getconst(ConstInt(-3))
     assert untag(tagged) == (-3, TAGINT)
-    const = ConstInt(50000)
+    const = ConstInt(5000000)
     tagged = memo.getconst(const)
     index, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert memo.consts[index - TAG_CONST_OFFSET] is const
-    tagged = memo.getconst(ConstInt(50000))
+    tagged = memo.getconst(ConstInt(5000000))
     index2, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert index2 == index
@@ -858,7 +858,7 @@
     base = [0, 0, tag(0, TAGBOX), tag(1, TAGINT),
             tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)]
 
-    assert unpack_numbering(numb) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                       tag(0, TAGBOX), tag(1, TAGINT)]
     t.append(0)
     snap2 = t.create_top_snapshot(FakeJitCode("jitcode", 0), 2, Frame(env2),
@@ -872,7 +872,7 @@
     assert numb_state2.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b3: tag(2, TAGBOX)}
     assert numb_state2.liveboxes is not numb_state.liveboxes
-    assert unpack_numbering(numb2) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb2) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                        tag(0, TAGBOX), tag(3, TAGINT)]
 
     t.append(0)
@@ -894,7 +894,7 @@
     assert numb_state3.num_virtuals == 0
     
     assert numb_state3.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
-    assert unpack_numbering(numb3) == ([0, 2, tag(3, TAGINT), tag(4, TAGINT),
+    assert unpack_numbering(numb3) == ([17, 0, 0, 2, tag(3, TAGINT), tag(4, TAGINT),
                                        tag(0, TAGBOX), tag(3, TAGINT)] +
                                        base + [0, 2])
 
@@ -911,7 +911,7 @@
     
     assert numb_state4.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL)}
-    assert unpack_numbering(numb4) == [0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
+    assert unpack_numbering(numb4) == [17, 0, 0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
                                        tag(0, TAGBOX), tag(3, TAGINT)] + base + [0, 2]
 
     t.append(0)
@@ -930,7 +930,7 @@
 
     assert numb_state5.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)}
-    assert unpack_numbering(numb5) == [
+    assert unpack_numbering(numb5) == [22, 0,
         3, tag(0, TAGBOX), tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL),
         0] + base + [
         2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT)
@@ -949,15 +949,17 @@
     numb_state = memo.number(FakeOptimizer(), 0, i)
     numb = numb_state.create_numbering()
     l = unpack_numbering(numb)
-    assert l[0] == 0
+    assert l[0] == len(l)
+    assert l[1] == 0
     assert l[1] == 0
     assert l[2] == 0
     assert l[3] == 0
+    assert l[4] == 0
     mapping = dict(zip(inpargs, i.inputargs))
     for i, item in enumerate(lst):
-        v, tag = untag(l[i + 4])
+        v, tag = untag(l[i + 6])
         if tag == TAGBOX:
-            assert l[i + 4] == numb_state.liveboxes[mapping[item]]
+            assert l[i + 6] == numb_state.liveboxes[mapping[item]]
         elif tag == TAGCONST:
             assert memo.consts[v].getint() == item.getint()
         elif tag == TAGINT:
@@ -1069,15 +1071,15 @@
     cpu = MyCPU([])
     reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe")
     reader.consume_vref_and_vable(None, None, None)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, sys.maxint, 1, sys.maxint, 2**16)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, 2, 3)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, sys.maxint, 2**16, -65)
 
 def test_virtual_adder_memo_const_sharing():
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**23), ConstInt(-65)]
     storage, t = make_storage(b1s, b2s, b3s)
     metainterp_sd = FakeMetaInterpStaticData()
     memo = ResumeDataLoopMemo(metainterp_sd)
@@ -1087,7 +1089,7 @@
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**24), ConstInt(-65)]
     storage2, t = make_storage(b1s, b2s, b3s)
     i = t.get_iter()
     modifier2 = ResumeDataVirtualAdder(FakeOptimizer(i), storage2, storage2,
@@ -1112,9 +1114,10 @@
                 return True
         class MyInfo:
             @staticmethod
-            def enumerate_vars(callback_i, callback_r, callback_f, _, index):
-                while index < len(self.numb.code):
-                    tagged, _ = resumecode.numb_next_item(self.numb, index)
+            def enumerate_vars(callback_i, callback_r, callback_f, _):
+                index = 0
+                while not self.done_reading():
+                    tagged = self.resumecodereader.peek()
                     _, tag = untag(tagged)
                     if tag == TAGVIRTUAL:
                         kind = REF
@@ -1122,20 +1125,21 @@
                         kind = Whatever()
                     box = self.decode_box(tagged, kind)
                     if box.type == INT:
-                        index = callback_i(index, index)
+                        callback_i(index)
                     elif box.type == REF:
-                        index = callback_r(index, index)
+                        callback_r(index)
                     elif box.type == FLOAT:
-                        index = callback_f(index, index)
+                        callback_f(index)
                     else:
                         assert 0
+                    index += 1
 
-        size, self.cur_index = resumecode.numb_next_item(self.numb, 0)
+        size = self.resumecodereader.next_item()
         assert size == 0
-        size, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
+        size = self.resumecodereader.next_item()
         assert size == 0
-        pc, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
-        jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
+        pc = self.resumecodereader.next_item()
+        jitcode_pos = self.resumecodereader.next_item()
 
         self._prepare_next_section(MyInfo())
         return self.lst
@@ -1228,7 +1232,7 @@
     liveboxes = []
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     b3t, b5t = [IntFrontendOp(0), RefFrontendOp(0)]
     b5t.setref_base(demo55o)
@@ -1299,7 +1303,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     b1t, b3t, b4t = [IntFrontendOp(0), IntFrontendOp(0), IntFrontendOp(0)]
     b1t.setint(11)
@@ -1352,7 +1356,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     b4t = RefFrontendOp(0)
     newboxes = _resume_remap(liveboxes, [#b2s -- virtual
                                          b4s], b4t)
@@ -1398,7 +1402,7 @@
     modifier._add_pending_fields(FakeOptimizer(), [
         ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)])
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     demo55.next = lltype.nullptr(LLtypeMixin.NODE)
     b2t = RefFrontendOp(0)
diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -1,29 +1,62 @@
-
-from rpython.jit.metainterp.resumecode import NUMBERING, NULL_NUMBER
 from rpython.jit.metainterp.resumecode import create_numbering,\
-    unpack_numbering
+    unpack_numbering, Reader, Writer
 from rpython.rtyper.lltypesystem import lltype
 
-from hypothesis import strategies, given
+from hypothesis import strategies, given, example
 
+examples = [
+    [1, 2, 3, 4, 257, 10000, 13, 15],
+    [1, 2, 3, 4],
+    range(1, 10, 2),
+    [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
+]
 
-def test_pack_unpack():
-    examples = [
-        [1, 2, 3, 4, 257, 10000, 13, 15],
-        [1, 2, 3, 4],
-        range(1, 10, 2),
-        [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
-    ]
-    for l in examples:
-        n = create_numbering(l)
-        assert unpack_numbering(n) == l
+def hypothesis_and_examples(func):
+    func = given(strategies.lists(strategies.integers(-2**15, 2**15-1)))(func)
+    for ex in examples:
+        func = example(ex)(func)
+    return func
 
- at given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+ at hypothesis_and_examples
 def test_roundtrip(l):
     n = create_numbering(l)
     assert unpack_numbering(n) == l
 
- at given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+ at hypothesis_and_examples
 def test_compressing(l):
     n = create_numbering(l)
     assert len(n.code) <= len(l) * 3
+
+ at hypothesis_and_examples
+def test_reader(l):
+    n = create_numbering(l)
+    r = Reader(n)
+    for i, elt in enumerate(l):
+        assert r.items_read == i
+        item = r.next_item()
+        assert elt == item
+
+ at hypothesis_and_examples
+def test_writer(l):
+    for size in [len(l), 0]:
+        w = Writer(len(l))
+        for num in l:
+            w.append_int(num)
+        n = w.create_numbering()
+        assert unpack_numbering(n) == l
+
+ at hypothesis_and_examples
+def test_patch(l):
+    for middle in range(len(l)):
+        l1 = l[:middle]
+        l2 = l[middle:]
+        w = Writer(len(l))
+        w.append_int(0)
+        for num in l1:
+            w.append_int(num)
+        w.patch_current_size(0)
+        for num in l2:
+            w.append_int(num)
+        n = w.create_numbering()
+        assert unpack_numbering(n)[1:] == l
+        assert unpack_numbering(n)[0] == middle + 1
diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -2,7 +2,6 @@
 from rpython.jit.metainterp import history
 from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem
 from rpython.jit.metainterp.warmstate import wrap, unwrap
-from rpython.jit.metainterp.resumecode import numb_next_item
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import rvirtualizable
 from rpython.rtyper.lltypesystem import lltype, llmemory
@@ -127,24 +126,21 @@
                 size += 1
             return size
 
-        def write_from_resume_data_partial(virtualizable, reader, index, numb):
+        def write_from_resume_data_partial(virtualizable, reader):
             virtualizable = cast_gcref_to_vtype(virtualizable)
             # Load values from the reader (see resume.py) described by
             # the list of numbers 'nums', and write them in their proper
             # place in the 'virtualizable'.
             for FIELDTYPE, fieldname in unroll_static_fields:
-                item, index = numb_next_item(numb, index)
-                x = reader.load_value_of_type(FIELDTYPE, item)
+                x = reader.load_next_value_of_type(FIELDTYPE)
                 setattr(virtualizable, fieldname, x)
             for ARRAYITEMTYPE, fieldname in unroll_array_fields:
                 lst = getattr(virtualizable, fieldname)
                 for j in range(getlength(lst)):
-                    item, index = numb_next_item(numb, index)                    
-                    x = reader.load_value_of_type(ARRAYITEMTYPE, item)
+                    x = reader.load_next_value_of_type(ARRAYITEMTYPE)
                     setarrayitem(lst, j, x)
-            return index
 
-        def load_list_of_boxes(virtualizable, reader, vable_box, numb, index):
+        def load_list_of_boxes(virtualizable, reader, vable_box):
             virtualizable = cast_gcref_to_vtype(virtualizable)
             # Uses 'virtualizable' only to know the length of the arrays;
             # does not write anything into it.  The returned list is in
@@ -152,17 +148,15 @@
             # the virtualizable itself.
             boxes = []
             for FIELDTYPE, fieldname in unroll_static_fields:
-                item, index = numb_next_item(numb, index)
-                box = reader.decode_box_of_type(FIELDTYPE, item)
+                box = reader.next_box_of_type(FIELDTYPE)
                 boxes.append(box)
             for ARRAYITEMTYPE, fieldname in unroll_array_fields:
                 lst = getattr(virtualizable, fieldname)
                 for j in range(getlength(lst)):
-                    item, index = numb_next_item(numb, index)                    
-                    box = reader.decode_box_of_type(ARRAYITEMTYPE, item)
+                    box = reader.next_box_of_type(ARRAYITEMTYPE)
                     boxes.append(box)
             boxes.append(vable_box)
-            return boxes, index
+            return boxes
 
         def check_boxes(virtualizable, boxes):
             virtualizable = cast_gcref_to_vtype(virtualizable)

From pypy.commits at gmail.com  Sun Feb 26 12:10:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 26 Feb 2017 09:10:53 -0800 (PST)
Subject: [pypy-commit] pypy default: (stevie, arigo)
Message-ID: <58b30c1d.4f072e0a.c5dc.9693@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90365:740872e8e466
Date: 2017-02-26 18:05 +0100
http://bitbucket.org/pypy/pypy/changeset/740872e8e466/

Log:	(stevie, arigo)

	Test and fix: outdentation errors (only!) were missing a 'filename'

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -725,6 +725,19 @@
         else:
             raise Exception("DID NOT RAISE")
 
+    def test_indent_error_filename(self):
+        source = py.code.Source("""
+        def f():
+          x
+         y
+        """)
+        try:
+            self.simple_test(source, None, None)
+        except IndentationError as e:
+            assert e.filename == '<test>'
+        else:
+            raise Exception("DID NOT RAISE")
+
     def test_kwargs_last(self):
         py.test.raises(SyntaxError, self.simple_test, "int(base=10, '2')",
                        None, None)
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -167,6 +167,9 @@
             except error.TokenError as e:
                 e.filename = compile_info.filename
                 raise
+            except error.TokenIndentationError as e:
+                e.filename = compile_info.filename
+                raise
             except parser.ParseError as e:
                 # Catch parse errors, pretty them up and reraise them as a
                 # SyntaxError.
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -77,7 +77,7 @@
         """)
         assert self.space.unwrap(w_args) == (
             'unindent does not match any outer indentation level',
-            (None, 3, 0, ' y\n'))
+            ('<string>', 3, 0, ' y\n'))
 
     def test_getcodeflags(self):
         code = self.compiler.compile('from __future__ import division\n',
@@ -1032,6 +1032,18 @@
         else:
             raise Exception("DID NOT RAISE")
 
+    def test_outdentation_error_filename(self):
+        source = """if 1:
+         x
+        y
+        """
+        try:
+            exec(source)
+        except IndentationError as e:
+            assert e.filename == '<string>'
+        else:
+            raise Exception("DID NOT RAISE")
+
     def test_repr_vs_str(self):
         source1 = "x = (\n"
         source2 = "x = (\n\n"

From pypy.commits at gmail.com  Sun Feb 26 12:10:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 26 Feb 2017 09:10:56 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58b30c20.95012e0a.99630.b521@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90366:4b4ca32b2aa3
Date: 2017-02-26 18:08 +0100
http://bitbucket.org/pypy/pypy/changeset/4b4ca32b2aa3/

Log:	merge heads

diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -280,8 +280,9 @@
                     # If L, V -> LV
                     current = SBase + ((current - LBase)*VCount + (next - VBase)) * TCount
                     continue
+                # Note: if next == TBase, leave LV unchanged
                 if (SBase <= current < SBase + SCount and
-                    TBase <= next < TBase + TCount and
+                    TBase < next < TBase + TCount and
                     (current - SBase) % TCount == 0):
                     # If LV, T -> LVT
                     current = current + (next - TBase)
diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/unicodedata/test/test_hyp.py
@@ -0,0 +1,47 @@
+import pytest
+try:
+    from hypothesis import given, strategies as st, example, settings
+except ImportError:
+    pytest.skip("hypothesis required")
+
+from pypy.module.unicodedata.interp_ucd import ucd
+
+def make_normalization(space, NF_code):
+    def normalize(s):
+        w_s = space.newunicode(s)
+        w_res = ucd.normalize(space, NF_code, w_s)
+        return space.unwrap(w_res)
+    return normalize
+
+all_forms = ['NFC', 'NFD', 'NFKC', 'NFKD']
+
+# For every (n1, n2, n3) triple, applying n1 then n2 must be the same
+# as applying n3.
+# Reference: http://unicode.org/reports/tr15/#Design_Goals
+compositions = [
+    ('NFC', 'NFC', 'NFC'),
+    ('NFC', 'NFD', 'NFD'),
+    ('NFC', 'NFKC', 'NFKC'),
+    ('NFC', 'NFKD', 'NFKD'),
+    ('NFD', 'NFC', 'NFC'),
+    ('NFD', 'NFD', 'NFD'),
+    ('NFD', 'NFKC', 'NFKC'),
+    ('NFD', 'NFKD', 'NFKD'),
+    ('NFKC', 'NFC', 'NFKC'),
+    ('NFKC', 'NFD', 'NFKD'),
+    ('NFKC', 'NFKC', 'NFKC'),
+    ('NFKC', 'NFKD', 'NFKD'),
+    ('NFKD', 'NFC', 'NFKC'),
+    ('NFKD', 'NFD', 'NFKD'),
+    ('NFKD', 'NFKC', 'NFKC'),
+    ('NFKD', 'NFKD', 'NFKD'),
+]
+
+
+ at pytest.mark.parametrize('NF1, NF2, NF3', compositions)
+ at example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at settings(max_examples=1000)
+ at given(s=st.text())
+def test_composition(s, space, NF1, NF2, NF3):
+    norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]]
+    assert norm2(norm1(s)) == norm3(s)

From pypy.commits at gmail.com  Sun Feb 26 12:10:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 26 Feb 2017 09:10:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58b30c22.12142e0a.cb56f.b6a7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r90367:4618cb6fc961
Date: 2017-02-26 18:10 +0100
http://bitbucket.org/pypy/pypy/changeset/4618cb6fc961/

Log:	hg merge default

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -156,19 +156,30 @@
 Does PyPy have a GIL?  Why?
 -------------------------------------------------
 
-Yes, PyPy has a GIL.  Removing the GIL is very hard.  The problems are
-essentially the same as with CPython (including the fact that our
-garbage collectors are not thread-safe so far).  Fixing it is possible,
-as shown by Jython and IronPython, but difficult.  It would require
-adapting the whole source code of PyPy, including subtle decisions about
-whether some effects are ok or not for the user (i.e. the Python
-programmer).
+Yes, PyPy has a GIL.  Removing the GIL is very hard.  On top of CPython,
+you have two problems:  (1) GC, in this case reference counting; (2) the
+whole Python language.
 
-Instead, since 2012, there is work going on on a still very experimental
-:doc:`Software Transactional Memory <stm>` (STM) version of PyPy.  This should give
-an alternative PyPy which works without a GIL, while at the same time
-continuing to give the Python programmer the complete illusion of having
-one.
+For PyPy, the hard issue is (2): by that I mean issues like what occurs
+if a mutable object is changed from one thread and read from another
+concurrently.  This is a problem for *any* mutable type: it needs
+careful review and fixes (fine-grained locks, mostly) through the
+*whole* Python interpreter.  It is a major effort, although not
+completely impossible, as Jython/IronPython showed.  This includes
+subtle decisions about whether some effects are ok or not for the user
+(i.e. the Python programmer).
+
+CPython has additionally the problem (1) of reference counting.  With
+PyPy, this sub-problem is simpler: we need to make our GC
+multithread-aware.  This is easier to do efficiently in PyPy than in
+CPython.  It doesn't solve the issue (2), though.
+
+Note that since 2012 there is work going on on a still very experimental
+:doc:`Software Transactional Memory <stm>` (STM) version of PyPy.  This
+should give an alternative PyPy which works without a GIL, while at the
+same time continuing to give the Python programmer the complete illusion
+of having one.  This work is currently a bit stalled because of its own
+technical difficulties.
 
 
 Is PyPy more clever than CPython about Tail Calls?
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -152,6 +152,13 @@
 behavior is also available on Python 2.x or for the ``dict`` type by
 calling ``__pypy__.move_to_end(dict, key, last=True)``.
 
+
+.. branch optinfo-into-bridges-3
+
+Improve the optimization of branchy Python code by retaining more information
+across failing guards.
+
+
 .. branch: space-newtext
 
 Internal refactoring of ``space.wrap()``, which is now replaced with
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -781,6 +781,19 @@
         else:
             raise Exception("DID NOT RAISE")
 
+    def test_indent_error_filename(self):
+        source = py.code.Source("""
+        def f():
+          x
+         y
+        """)
+        try:
+            self.simple_test(source, None, None)
+        except IndentationError as e:
+            assert e.filename == '<test>'
+        else:
+            raise Exception("DID NOT RAISE")
+
     def test_kwargs_last(self):
         py.test.raises(SyntaxError, self.simple_test, "int(base=10, '2')",
                        None, None)
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -205,6 +205,9 @@
             except error.TokenError as e:
                 e.filename = compile_info.filename
                 raise
+            except error.TokenIndentationError as e:
+                e.filename = compile_info.filename
+                raise
             except parser.ParseError as e:
                 # Catch parse errors, pretty them up and reraise them as a
                 # SyntaxError.
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -77,7 +77,7 @@
         """)
         assert self.space.unwrap(w_args) == (
             'unindent does not match any outer indentation level',
-            (None, 3, 0, ' y\n'))
+            ('<string>', 3, 0, ' y\n'))
 
     def test_getcodeflags(self):
         code = self.compiler.compile('from __future__ import division\n',
@@ -1334,6 +1334,18 @@
         else:
             raise Exception("DID NOT RAISE")
 
+    def test_outdentation_error_filename(self):
+        source = """if 1:
+         x
+        y
+        """
+        try:
+            exec(source)
+        except IndentationError as e:
+            assert e.filename == '<string>'
+        else:
+            raise Exception("DID NOT RAISE")
+
     def test_taberror(self):
         source = """if 1:
         x
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -289,8 +289,9 @@
                     # If L, V -> LV
                     current = SBase + ((current - LBase)*VCount + (next - VBase)) * TCount
                     continue
+                # Note: if next == TBase, leave LV unchanged
                 if (SBase <= current < SBase + SCount and
-                    TBase <= next < TBase + TCount and
+                    TBase < next < TBase + TCount and
                     (current - SBase) % TCount == 0):
                     # If LV, T -> LVT
                     current = current + (next - TBase)
diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/unicodedata/test/test_hyp.py
@@ -0,0 +1,47 @@
+import pytest
+try:
+    from hypothesis import given, strategies as st, example, settings
+except ImportError:
+    pytest.skip("hypothesis required")
+
+from pypy.module.unicodedata.interp_ucd import ucd
+
+def make_normalization(space, NF_code):
+    def normalize(s):
+        w_s = space.newunicode(s)
+        w_res = ucd.normalize(space, NF_code, w_s)
+        return space.unwrap(w_res)
+    return normalize
+
+all_forms = ['NFC', 'NFD', 'NFKC', 'NFKD']
+
+# For every (n1, n2, n3) triple, applying n1 then n2 must be the same
+# as applying n3.
+# Reference: http://unicode.org/reports/tr15/#Design_Goals
+compositions = [
+    ('NFC', 'NFC', 'NFC'),
+    ('NFC', 'NFD', 'NFD'),
+    ('NFC', 'NFKC', 'NFKC'),
+    ('NFC', 'NFKD', 'NFKD'),
+    ('NFD', 'NFC', 'NFC'),
+    ('NFD', 'NFD', 'NFD'),
+    ('NFD', 'NFKC', 'NFKC'),
+    ('NFD', 'NFKD', 'NFKD'),
+    ('NFKC', 'NFC', 'NFKC'),
+    ('NFKC', 'NFD', 'NFKD'),
+    ('NFKC', 'NFKC', 'NFKC'),
+    ('NFKC', 'NFKD', 'NFKD'),
+    ('NFKD', 'NFC', 'NFKC'),
+    ('NFKD', 'NFD', 'NFKD'),
+    ('NFKD', 'NFKC', 'NFKC'),
+    ('NFKD', 'NFKD', 'NFKD'),
+]
+
+
+ at pytest.mark.parametrize('NF1, NF2, NF3', compositions)
+ at example(s=u'---\uafb8\u11a7---')  # issue 2289
+ at settings(max_examples=1000)
+ at given(s=st.text())
+def test_composition(s, space, NF1, NF2, NF3):
+    norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]]
+    assert norm2(norm1(s)) == norm3(s)
diff --git a/rpython/jit/codewriter/jitcode.py b/rpython/jit/codewriter/jitcode.py
--- a/rpython/jit/codewriter/jitcode.py
+++ b/rpython/jit/codewriter/jitcode.py
@@ -146,14 +146,13 @@
     def get_register_index_f(self, index):
         return ord(self.live_f[index])
 
-    def enumerate_vars(self, callback_i, callback_r, callback_f, spec, index):
+    def enumerate_vars(self, callback_i, callback_r, callback_f, spec):
         for i in range(self.get_register_count_i()):
-            index = callback_i(index, self.get_register_index_i(i))
+            callback_i(self.get_register_index_i(i))
         for i in range(self.get_register_count_r()):
-            index = callback_r(index, self.get_register_index_r(i))
+            callback_r(self.get_register_index_r(i))
         for i in range(self.get_register_count_f()):
-            index = callback_f(index, self.get_register_index_f(i))
-        return index
+            callback_f(self.get_register_index_f(i))
     enumerate_vars._annspecialcase_ = 'specialize:arg(4)'
 
 _liveness_cache = {}
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -85,13 +85,14 @@
     """ This represents ops() with a jump at the end that goes to some
     loop, we need to deal with virtual state and inlining of short preamble
     """
-    def __init__(self, trace, runtime_boxes, call_pure_results=None,
+    def __init__(self, trace, runtime_boxes, resumestorage=None, call_pure_results=None,
                  enable_opts=None, inline_short_preamble=False):
         self.trace = trace
         self.runtime_boxes = runtime_boxes
         self.call_pure_results = call_pure_results
         self.enable_opts = enable_opts
         self.inline_short_preamble = inline_short_preamble
+        self.resumestorage = resumestorage
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -100,7 +101,8 @@
         return opt.optimize_bridge(self.trace, self.runtime_boxes,
                                    self.call_pure_results,
                                    self.inline_short_preamble,
-                                   self.box_names_memo)
+                                   self.box_names_memo,
+                                   self.resumestorage)
 
 class UnrolledLoopData(CompileData):
     """ This represents label() ops jump with extra info that's from the
@@ -870,10 +872,9 @@
 
 
 class ResumeGuardDescr(AbstractResumeGuardDescr):
-    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+    _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals',
                'rd_pendingfields', 'status')
     rd_numb = lltype.nullptr(NUMBERING)
-    rd_count = 0
     rd_consts = None
     rd_virtuals = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
@@ -882,7 +883,6 @@
         if isinstance(other, ResumeGuardCopiedDescr):
             other = other.prev
         assert isinstance(other, ResumeGuardDescr)
-        self.rd_count = other.rd_count
         self.rd_consts = other.rd_consts
         self.rd_pendingfields = other.rd_pendingfields
         self.rd_virtuals = other.rd_virtuals
@@ -895,7 +895,6 @@
 
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
-        self.rd_count = len(boxes)
         self.store_hash(metainterp_sd)
 
     def clone(self):
@@ -1077,7 +1076,15 @@
     call_pure_results = metainterp.call_pure_results
 
     if metainterp.history.ends_with_jump:
-        data = BridgeCompileData(trace, runtime_boxes,
+        if isinstance(resumekey, ResumeGuardCopiedDescr):
+            key = resumekey.prev
+            assert isinstance(key, ResumeGuardDescr)
+        elif isinstance(resumekey, ResumeFromInterpDescr):
+            key = None
+        else:
+            key = resumekey
+            assert isinstance(key, ResumeGuardDescr)
+        data = BridgeCompileData(trace, runtime_boxes, key,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts,
                                  inline_short_preamble=inline_short_preamble)
diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py
--- a/rpython/jit/metainterp/opencoder.py
+++ b/rpython/jit/metainterp/opencoder.py
@@ -66,7 +66,7 @@
         assert isinstance(snapshot, TopSnapshot)
         self.vable_array = snapshot.vable_array
         self.vref_array = snapshot.vref_array
-        self.size = len(self.vable_array) + len(self.vref_array) + 2
+        self.size = len(self.vable_array) + len(self.vref_array) + 3
         jc_index, pc = unpack_uint(snapshot.packed_jitcode_pc)
         self.framestack = []
         if jc_index == 2**16-1:
diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -0,0 +1,135 @@
+""" Code to feed information from the optimizer via the resume code into the
+optimizer of the bridge attached to a guard. """
+
+from rpython.jit.metainterp import resumecode
+
+
+# adds the following sections at the end of the resume code:
+#
+# ---- known classes
+# <bitfield> size is the number of reference boxes in the liveboxes
+#            1 klass known
+#            0 klass unknown
+#            (the class is found by actually looking at the runtime value)
+#            the bits are bunched in bunches of 7
+#
+# ---- heap knowledge
+# <length>
+# (<box1> <descr> <box2>) length times, if getfield(box1, descr) == box2
+#                         both boxes should be in the liveboxes
+#
+# ----
+
+
+# maybe should be delegated to the optimization classes?
+
+def tag_box(box, liveboxes_from_env, memo):
+    from rpython.jit.metainterp.history import Const
+    if isinstance(box, Const):
+        return memo.getconst(box)
+    else:
+        return liveboxes_from_env[box] # has to exist
+
+def decode_box(resumestorage, tagged, liveboxes, cpu):
+    from rpython.jit.metainterp.resume import untag, TAGCONST, TAGINT, TAGBOX
+    from rpython.jit.metainterp.resume import NULLREF, TAG_CONST_OFFSET, tagged_eq
+    from rpython.jit.metainterp.history import ConstInt
+    num, tag = untag(tagged)
+    # NB: the TAGVIRTUAL case can't happen here, because this code runs after
+    # virtuals are already forced again
+    if tag == TAGCONST:
+        if tagged_eq(tagged, NULLREF):
+            box = cpu.ts.CONST_NULL
+        else:
+            box = resumestorage.rd_consts[num - TAG_CONST_OFFSET]
+    elif tag == TAGINT:
+        box = ConstInt(num)
+    elif tag == TAGBOX:
+        box = liveboxes[num]
+    else:
+        raise AssertionError("unreachable")
+    return box
+
+def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo):
+    available_boxes = {}
+    for box in liveboxes:
+        if box is not None and box in liveboxes_from_env:
+            available_boxes[box] = None
+    metainterp_sd = optimizer.metainterp_sd
+
+    # class knowledge is stored as bits, true meaning the class is known, false
+    # means unknown. on deserializing we look at the bits, and read the runtime
+    # class for the known classes (which has to be the same in the bridge) and
+    # mark that as known. this works for guard_class too: the class is only
+    # known *after* the guard
+    bitfield = 0
+    shifts = 0
+    for box in liveboxes:
+        if box is None or box.type != "r":
+            continue
+        info = optimizer.getptrinfo(box)
+        known_class = info is not None and info.get_known_class(optimizer.cpu) is not None
+        bitfield <<= 1
+        bitfield |= known_class
+        shifts += 1
+        if shifts == 6:
+            numb_state.append_int(bitfield)
+            bitfield = shifts = 0
+    if shifts:
+        numb_state.append_int(bitfield << (6 - shifts))
+
+    # heap knowledge: we store triples of known heap fields in non-virtual
+    # structs
+    # XXX could be extended to arrays
+    if optimizer.optheap:
+        triples = optimizer.optheap.serialize_optheap(available_boxes)
+        # can only encode descrs that have a known index into
+        # metainterp_sd.all_descrs
+        triples = [triple for triple in triples if triple[1].descr_index != -1]
+        numb_state.append_int(len(triples))
+        for box1, descr, box2 in triples:
+            index = descr.descr_index
+            numb_state.append_short(tag_box(box1, liveboxes_from_env, memo))
+            numb_state.append_int(index)
+            numb_state.append_short(tag_box(box2, liveboxes_from_env, memo))
+    else:
+        numb_state.append_int(0)
+
+def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes):
+    reader = resumecode.Reader(resumestorage.rd_numb)
+    assert len(frontend_boxes) == len(liveboxes)
+    metainterp_sd = optimizer.metainterp_sd
+
+    # skip resume section
+    startcount = reader.next_item()
+    reader.jump(startcount - 1)
+
+    # class knowledge
+    bitfield = 0
+    mask = 0
+    for i, box in enumerate(liveboxes):
+        if box.type != "r":
+            continue
+        if not mask:
+            bitfield = reader.next_item()
+            mask = 0b100000
+        class_known = bitfield & mask
+        mask >>= 1
+        if class_known:
+            cls = optimizer.cpu.ts.cls_of_box(frontend_boxes[i])
+            optimizer.make_constant_class(box, cls)
+
+    # heap knowledge
+    if not optimizer.optheap:
+        return
+    length = reader.next_item()
+    result = []
+    for i in range(length):
+        tagged = reader.next_item()
+        box1 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
+        index = reader.next_item()
+        descr = metainterp_sd.all_descrs[index]
+        tagged = reader.next_item()
+        box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
+        result.append((box1, descr, box2))
+    optimizer.optheap.deserialize_optheap(result)
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -48,6 +48,7 @@
         # that has a non-None entry at
         # info._fields[descr.get_index()]
         # must be in cache_infos
+        assert structop.type == 'r'
         self.cached_structs.append(structop)
         self.cached_infos.append(info)
 
@@ -693,6 +694,36 @@
         self._seen_guard_not_invalidated = True
         return self.emit(op)
 
+    def serialize_optheap(self, available_boxes):
+        result = []
+        for descr, cf in self.cached_fields.iteritems():
+            if cf._lazy_set:
+                continue # XXX safe default for now
+            parent_descr = descr.get_parent_descr()
+            if not parent_descr.is_object():
+                continue # XXX could be extended to non-instance objects
+            for i, box1 in enumerate(cf.cached_structs):
+                if box1 not in available_boxes:
+                    continue
+                structinfo = cf.cached_infos[i]
+                box2 = structinfo.getfield(descr).get_box_replacement()
+                if isinstance(box2, Const) or box2 in available_boxes:
+                    result.append((box1, descr, box2))
+        return result
+
+    def deserialize_optheap(self, triples):
+        for box1, descr, box2 in triples:
+            parent_descr = descr.get_parent_descr()
+            assert parent_descr.is_object()
+            structinfo = box1.get_forwarded()
+            if not isinstance(structinfo, info.AbstractVirtualPtrInfo):
+                structinfo = info.InstancePtrInfo(parent_descr)
+                structinfo.init_fields(parent_descr, descr.get_index())
+                box1.set_forwarded(structinfo)
+
+            cf = self.field_cache(descr)
+            structinfo.setfield(descr, box1, box2, optheap=self, cf=cf)
+
 
 dispatch_opt = make_dispatcher_method(OptHeap, 'optimize_',
                                       default=OptHeap.emit)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -70,7 +70,7 @@
     def same_info(self, other):
         return self is other
 
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+    def getstrlen(self, op, string_optimizer, mode):
         return None
 
     def getstrhash(self, op, mode):
@@ -777,21 +777,20 @@
             # XXX we can do better if we know it's an array
             return IntLowerBound(0)
         else:
-            return ConstIntBound(self.getstrlen(None, None, mode).getint())
-    
-    def getstrlen(self, op, string_optimizer, mode, create_ops=True):
+            return ConstIntBound(self.getstrlen1(mode))
+
+    def getstrlen(self, op, string_optimizer, mode):
+        return ConstInt(self.getstrlen1(mode))
+
+    def getstrlen1(self, mode):
         from rpython.jit.metainterp.optimizeopt import vstring
         
         if mode is vstring.mode_string:
             s = self._unpack_str(vstring.mode_string)
-            if s is None:
-                return None
-            return ConstInt(len(s))
+            return len(s)
         else:
             s = self._unpack_str(vstring.mode_unicode)            
-            if s is None:
-                return None
-            return ConstInt(len(s))
+            return len(s)
 
     def getstrhash(self, op, mode):
         from rpython.jit.metainterp.optimizeopt import vstring
@@ -812,7 +811,7 @@
         from rpython.jit.metainterp.optimizeopt import vstring
         from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
 
-        lgt = self.getstrlen(op, string_optimizer, mode, False)
+        lgt = self.getstrlen(op, string_optimizer, mode)
         return vstring.copy_str_content(string_optimizer, self._const,
                                         targetbox, CONST_0, offsetbox,
                                         lgt, mode)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -297,6 +297,7 @@
         self.optrewrite = None
         self.optearlyforce = None
         self.optunroll = None
+        self._really_emitted_operation = None
 
         self._last_guard_op = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -30,6 +30,7 @@
         self.add_guard_future_condition(bridge)
         trace = oparser.convert_loop_to_trace(bridge, FakeMetaInterpStaticData(self.cpu))
         data = compile.BridgeCompileData(trace, self.convert_values(bridge.operations[-1].getarglist(), bridge_values),
+                                         None,
                                          enable_opts=self.enable_opts,
                             inline_short_preamble=inline_short_preamble)
         bridge_info, ops = self._do_optimize_loop(data)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -232,9 +232,15 @@
         return label_vs
 
     def optimize_bridge(self, trace, runtime_boxes, call_pure_results,
-                        inline_short_preamble, box_names_memo):
+                        inline_short_preamble, box_names_memo, resumestorage):
+        from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+        frontend_inputargs = trace.inputargs
         trace = trace.get_iter()
         self._check_no_forwarding([trace.inputargs])
+        if resumestorage:
+            deserialize_optimizer_knowledge(self.optimizer,
+                                            resumestorage, frontend_inputargs,
+                                            trace.inputargs)
         info, ops = self.optimizer.propagate_all_forward(trace,
             call_pure_results, False)
         jump_op = info.jump_op
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -752,7 +752,6 @@
 
     def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)):
         self.setref_base(r)
-        self.datatype = 'r'
 
     def reset_value(self):
         self.setref_base(lltype.nullptr(llmemory.GCREF.TO))
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -162,25 +162,13 @@
 UNINITIALIZED = tag(-2, TAGCONST)   # used for uninitialized string characters
 TAG_CONST_OFFSET = 0
 
-class NumberingState(object):
+class NumberingState(resumecode.Writer):
     def __init__(self, size):
+        resumecode.Writer.__init__(self, size)
         self.liveboxes = {}
-        self.current = [rffi.cast(rffi.SHORT, 0)] * size
-        self._pos = 0
         self.num_boxes = 0
         self.num_virtuals = 0
 
-    def append_short(self, item):
-        self.current[self._pos] = item
-        self._pos += 1
-
-    def append_int(self, item):
-        short = rffi.cast(rffi.SHORT, item)
-        assert rffi.cast(lltype.Signed, short) == item
-        return self.append_short(short)
-
-    def create_numbering(self):
-        return resumecode.create_numbering(self.current)
 
 class ResumeDataLoopMemo(object):
 
@@ -268,6 +256,8 @@
     def number(self, optimizer, position, trace):
         snapshot_iter = trace.get_snapshot_iter(position)
         numb_state = NumberingState(snapshot_iter.size)
+        numb_state.append_int(0) # patch later: size of resume section
+        numb_state.append_int(0) # patch later: number of failargs
 
         arr = snapshot_iter.vable_array
 
@@ -287,6 +277,7 @@
             numb_state.append_int(pc)
             self._number_boxes(
                     snapshot_iter, snapshot.box_array, optimizer, numb_state)
+        numb_state.patch_current_size(0)
 
         return numb_state
 
@@ -471,6 +462,9 @@
         self._number_virtuals(liveboxes, optimizer, num_virtuals)
         self._add_pending_fields(optimizer, pending_setfields)
 
+        numb_state.patch(1, len(liveboxes))
+
+        self._add_optimizer_sections(numb_state, liveboxes, liveboxes_from_env)
         storage.rd_numb = numb_state.create_numbering()
         storage.rd_consts = self.memo.consts
         return liveboxes[:]
@@ -590,6 +584,12 @@
                 return self.liveboxes_from_env[box]
             return self.liveboxes[box]
 
+    def _add_optimizer_sections(self, numb_state, liveboxes, liveboxes_from_env):
+        # add extra information about things the optimizer learned
+        from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge
+        serialize_optimizer_knowledge(
+            self.optimizer, numb_state, liveboxes, liveboxes_from_env, self.memo)
+
 class AbstractVirtualInfo(object):
     kind = REF
     is_about_raw = False
@@ -931,9 +931,10 @@
 
     def _init(self, cpu, storage):
         self.cpu = cpu
-        self.numb = storage.rd_numb
-        self.cur_index = 0
-        self.count = storage.rd_count
+        self.resumecodereader = resumecode.Reader(storage.rd_numb)
+        items_resume_section = self.resumecodereader.next_item()
+        self.items_resume_section = items_resume_section
+        self.count = self.resumecodereader.next_item()
         self.consts = storage.rd_consts
 
     def _prepare(self, storage):
@@ -941,14 +942,21 @@
         self._prepare_pendingfields(storage.rd_pendingfields)
 
     def read_jitcode_pos_pc(self):
-        jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb,
-            self.cur_index)
-        pc, self.cur_index = resumecode.numb_next_item(self.numb,
-            self.cur_index)
+        jitcode_pos = self.resumecodereader.next_item()
+        pc = self.resumecodereader.next_item()
         return jitcode_pos, pc
 
+    def next_int(self):
+        return self.decode_int(self.resumecodereader.next_item())
+
+    def next_ref(self):
+        return self.decode_ref(self.resumecodereader.next_item())
+
+    def next_float(self):
+        return self.decode_float(self.resumecodereader.next_item())
+
     def done_reading(self):
-        return self.cur_index >= len(self.numb.code)
+        return self.resumecodereader.items_read >= self.items_resume_section
 
     def getvirtual_ptr(self, index):
         # Returns the index'th virtual, building it lazily if needed.
@@ -1025,29 +1033,22 @@
     def _prepare_next_section(self, info):
         # Use info.enumerate_vars(), normally dispatching to
         # rpython.jit.codewriter.jitcode.  Some tests give a different 'info'.
-        self.cur_index = info.enumerate_vars(self._callback_i,
-                                        self._callback_r,
-                                        self._callback_f,
-                                        self.unique_id,  # <-- annotation hack
-                                        self.cur_index)
+        info.enumerate_vars(self._callback_i,
+                            self._callback_r,
+                            self._callback_f,
+                            self.unique_id)  # <-- annotation hack
 
-    def _callback_i(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_int(item)
+    def _callback_i(self, register_index):
+        value = self.next_int()
         self.write_an_int(register_index, value)
-        return index
 
-    def _callback_r(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_ref(item)
+    def _callback_r(self, register_index):
+        value = self.next_ref()
         self.write_a_ref(register_index, value)
-        return index
 
-    def _callback_f(self, index, register_index):
-        item, index = resumecode.numb_next_item(self.numb, index)
-        value = self.decode_float(item)
+    def _callback_f(self, register_index):
+        value = self.next_float()
         self.write_a_float(register_index, value)
-        return index
 
 # ---------- when resuming for pyjitpl.py, make boxes ----------
 
@@ -1057,6 +1058,7 @@
     boxes = resumereader.consume_vref_and_vable_boxes(virtualizable_info,
                                                       greenfield_info)
     virtualizable_boxes, virtualref_boxes = boxes
+
     while not resumereader.done_reading():
         jitcode_pos, pc = resumereader.read_jitcode_pos_pc()
         jitcode = metainterp.staticdata.jitcodes[jitcode_pos]
@@ -1076,7 +1078,7 @@
         self._init(metainterp.cpu, storage)
         self.deadframe = deadframe
         self.metainterp = metainterp
-        self.liveboxes = [None] * storage.rd_count
+        self.liveboxes = [None] * self.count
         self._prepare(storage)
 
     def consume_boxes(self, info, boxes_i, boxes_r, boxes_f):
@@ -1085,42 +1087,30 @@
         self.boxes_f = boxes_f
         self._prepare_next_section(info)
 
-    def consume_virtualizable_boxes(self, vinfo, index):
+    def consume_virtualizable_boxes(self, vinfo):
         # we have to ignore the initial part of 'nums' (containing vrefs),
         # find the virtualizable from nums[-1], and use it to know how many
         # boxes of which type we have to return.  This does not write
         # anything into the virtualizable.
-        numb = self.numb
-        item, index = resumecode.numb_next_item(numb, index)
-        virtualizablebox = self.decode_ref(item)
+        virtualizablebox = self.next_ref()
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox)
-        return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox,
-            numb, index)
+        return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox)
 
-    def consume_virtualref_boxes(self, index):
+    def consume_virtualref_boxes(self):
         # Returns a list of boxes, assumed to be all BoxPtrs.
         # We leave up to the caller to call vrefinfo.continue_tracing().
-        size, index = resumecode.numb_next_item(self.numb, index)
-        if size == 0:
-            return [], index
-        lst = []
-        for i in range(size * 2):
-            item, index = resumecode.numb_next_item(self.numb, index)
-            lst.append(self.decode_ref(item))
-        return lst, index
+        size = self.resumecodereader.next_item()
+        return [self.next_ref() for i in range(size * 2)]
 
     def consume_vref_and_vable_boxes(self, vinfo, ginfo):
-        vable_size, index = resumecode.numb_next_item(self.numb, 0)
+        vable_size = self.resumecodereader.next_item()
         if vinfo is not None:
-            virtualizable_boxes, index = self.consume_virtualizable_boxes(vinfo,
-                                                                          index)
+            virtualizable_boxes = self.consume_virtualizable_boxes(vinfo)
         elif ginfo is not None:
-            item, index = resumecode.numb_next_item(self.numb, index)
-            virtualizable_boxes = [self.decode_ref(item)]
+            virtualizable_boxes = [self.next_ref()]
         else:
             virtualizable_boxes = None
-        virtualref_boxes, index = self.consume_virtualref_boxes(index)
-        self.cur_index = index
+        virtualref_boxes = self.consume_virtualref_boxes()
         return virtualizable_boxes, virtualref_boxes
 
     def allocate_with_vtable(self, descr=None):
@@ -1297,7 +1287,7 @@
         self.liveboxes[num] = box
         return box
 
-    def decode_box_of_type(self, TYPE, tagged):
+    def next_box_of_type(self, TYPE):
         kind = getkind(TYPE)
         if kind == 'int':
             kind = INT
@@ -1307,8 +1297,8 @@
             kind = FLOAT
         else:
             raise AssertionError(kind)
-        return self.decode_box(tagged, kind)
-    decode_box_of_type._annspecialcase_ = 'specialize:arg(1)'
+        return self.decode_box(self.resumecodereader.next_item(), kind)
+    next_box_of_type._annspecialcase_ = 'specialize:arg(1)'
 
     def write_an_int(self, index, box):
         self.boxes_i[index] = box
@@ -1397,64 +1387,54 @@
         info = blackholeinterp.get_current_position_info()
         self._prepare_next_section(info)
 
-    def consume_virtualref_info(self, vrefinfo, index):
+    def consume_virtualref_info(self, vrefinfo):
         # we have to decode a list of references containing pairs
         # [..., virtual, vref, ...] and returns the index at the end
-        size, index = resumecode.numb_next_item(self.numb, index)
+        size = self.resumecodereader.next_item()
         if vrefinfo is None or size == 0:
             assert size == 0
-            return index
+            return
         for i in range(size):
-            virtual_item, index = resumecode.numb_next_item(
-                self.numb, index)
-            vref_item, index = resumecode.numb_next_item(
-                self.numb, index)
-            virtual = self.decode_ref(virtual_item)
-            vref = self.decode_ref(vref_item)
+            virtual = self.next_ref()
+            vref = self.next_ref()
             # For each pair, we store the virtual inside the vref.
             vrefinfo.continue_tracing(vref, virtual)
-        return index
 
-    def consume_vable_info(self, vinfo, index):
+    def consume_vable_info(self, vinfo):
         # we have to ignore the initial part of 'nums' (containing vrefs),
         # find the virtualizable from nums[-1], load all other values
         # from the CPU stack, and copy them into the virtualizable
-        numb = self.numb
-        item, index = resumecode.numb_next_item(self.numb, index)
-        virtualizable = self.decode_ref(item)
+        virtualizable = self.next_ref()
         # just reset the token, we'll force it later
         vinfo.reset_token_gcref(virtualizable)
-        index = vinfo.write_from_resume_data_partial(virtualizable, self,
-            index, numb)
-        return index
+        vinfo.write_from_resume_data_partial(virtualizable, self)
 
-    def load_value_of_type(self, TYPE, tagged):
+    def load_next_value_of_type(self, TYPE):
         from rpython.jit.metainterp.warmstate import specialize_value
         kind = getkind(TYPE)
         if kind == 'int':
-            x = self.decode_int(tagged)
+            x = self.next_int()
         elif kind == 'ref':
-            x = self.decode_ref(tagged)
+            x = self.next_ref()
         elif kind == 'float':
-            x = self.decode_float(tagged)
+            x = self.next_float()
         else:
             raise AssertionError(kind)
         return specialize_value(TYPE, x)
-    load_value_of_type._annspecialcase_ = 'specialize:arg(1)'
+    load_next_value_of_type._annspecialcase_ = 'specialize:arg(1)'
 
     def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo):
-        vable_size, index = resumecode.numb_next_item(self.numb, 0)
+        vable_size = self.resumecodereader.next_item()
         if self.resume_after_guard_not_forced != 2:
             if vinfo is not None:
-                index = self.consume_vable_info(vinfo, index)
+                self.consume_vable_info(vinfo)
             if ginfo is not None:
-                _, index = resumecode.numb_next_item(self.numb, index)
-            index = self.consume_virtualref_info(vrefinfo, index)
+                _ = self.resumecodereader.next_item()
+            self.consume_virtualref_info(vrefinfo)
         else:
-            index = resumecode.numb_next_n_items(self.numb, vable_size, index)
-            vref_size, index = resumecode.numb_next_item(self.numb, index)
-            index = resumecode.numb_next_n_items(self.numb, vref_size * 2, index)
-        self.cur_index = index 
+            self.resumecodereader.jump(vable_size)
+            vref_size = self.resumecodereader.next_item()
+            self.resumecodereader.jump(vref_size * 2)
 
     def allocate_with_vtable(self, descr=None):
         from rpython.jit.metainterp.executor import exec_new_with_vtable
diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py
--- a/rpython/jit/metainterp/resumecode.py
+++ b/rpython/jit/metainterp/resumecode.py
@@ -1,6 +1,9 @@
 
 """ Resume bytecode. It goes as following:
 
+  # ----- resume section
+  [total size of resume section]
+  [number of failargs]
   [<length> <virtualizable object> <numb> <numb> <numb>]    if vinfo is not None
    -OR-
   [1 <ginfo object>]                                        if ginfo is not None
@@ -13,10 +16,14 @@
   [<pc> <jitcode> <numb> <numb>]
   ...
 
-  until the length of the array.
+  until the size of the resume section
+
+  # ----- optimization section
+  <more code>                                      further sections according to bridgeopt.py
 """
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib import objectmodel
 
 NUMBERINGP = lltype.Ptr(lltype.GcForwardReference())
 NUMBERING = lltype.GcStruct('Numbering',
@@ -24,33 +31,24 @@
 NUMBERINGP.TO.become(NUMBERING)
 NULL_NUMBER = lltype.nullptr(NUMBERING)
 
-def create_numbering(lst, total=-1):
-    if total == -1:
-        total = len(lst)
-    result = []
-    for i in range(total):
-        item = lst[i]
-        item = rffi.cast(lltype.Signed, item)
-        item *= 2
-        if item < 0:
-            item = -1 - item
+def append_numbering(lst, item):
+    item = rffi.cast(lltype.Signed, item)
+    item *= 2
+    if item < 0:
+        item = -1 - item
 
-        assert item >= 0
-        if item < 2**7:
-            result.append(rffi.cast(rffi.UCHAR, item))
-        elif item < 2**14:
-            result.append(rffi.cast(rffi.UCHAR, item | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, item >> 7))
-        else:
-            assert item < 2**16
-            result.append(rffi.cast(rffi.UCHAR, item | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
-            result.append(rffi.cast(rffi.UCHAR, item >> 14))
+    assert item >= 0
+    if item < 2**7:
+        lst.append(rffi.cast(rffi.UCHAR, item))
+    elif item < 2**14:
+        lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, item >> 7))
+    else:
+        assert item < 2**16
+        lst.append(rffi.cast(rffi.UCHAR, item | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80))
+        lst.append(rffi.cast(rffi.UCHAR, item >> 14))
 
-    numb = lltype.malloc(NUMBERING, len(result))
-    for i in range(len(result)):
-        numb.code[i] = result[i]
-    return numb
 
 def numb_next_item(numb, index):
     value = rffi.cast(lltype.Signed, numb.code[index])
@@ -81,3 +79,64 @@
         next, i = numb_next_item(numb, i)
         l.append(next)
     return l
+
+class Writer(object):
+    def __init__(self, size=0):
+        self.current = objectmodel.newlist_hint(size)
+
+    def append_short(self, item):
+        self.current.append(item)
+
+    def append_int(self, item):
+        short = rffi.cast(rffi.SHORT, item)
+        assert rffi.cast(lltype.Signed, short) == item
+        return self.append_short(short)
+
+    def create_numbering(self):
+        final = objectmodel.newlist_hint(len(self.current) * 3)
+        for item in self.current:
+            append_numbering(final, item)
+        numb = lltype.malloc(NUMBERING, len(final))
+        for i, elt in enumerate(final):
+            numb.code[i] = elt
+        return numb
+
+    def patch_current_size(self, index):
+        self.patch(index, len(self.current))
+
+    def patch(self, index, item):
+        self.current[index] = item
+
+def create_numbering(l):
+    w = Writer()
+    for item in l:
+        w.append_int(item)
+    return w.create_numbering()
+
+
+class Reader(object):
+    def __init__(self, code):
+        self.code = code
+        self.cur_pos = 0 # index into the code
+        self.items_read = 0 # number of items read
+
+    def next_item(self):
+        result, self.cur_pos = numb_next_item(self.code, self.cur_pos)
+        self.items_read += 1
+        return result
+
+    def peek(self):
+        result, _ = numb_next_item(self.code, self.cur_pos)
+        return result
+
+    def jump(self, size):
+        """ jump n items forward without returning anything """
+        index = self.cur_pos
+        for i in range(size):
+            _, index = numb_next_item(self.code, index)
+        self.items_read += size
+        self.cur_pos = index
+
+    def unpack(self):
+        # mainly for debugging
+        return unpack_numbering(self.code)
diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_bridgeopt.py
@@ -0,0 +1,188 @@
+# tests that check that information is fed from the optimizer into the bridges
+
+import math
+from rpython.rlib import jit
+from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge
+from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
+from rpython.jit.metainterp.resoperation import InputArgRef, InputArgInt
+from rpython.jit.metainterp.resume import NumberingState
+from rpython.jit.metainterp.resumecode import unpack_numbering
+from rpython.jit.metainterp.optimizeopt.info import InstancePtrInfo
+
+from hypothesis import strategies, given
+
+class FakeTS(object):
+    def __init__(self, dct):
+        self.dct = dct
+
+    def cls_of_box(self, box):
+        return self.dct[box]
+
+
+class FakeCPU(object):
+    def __init__(self, dct):
+        self.ts = FakeTS(dct)
+
+class FakeOptimizer(object):
+    metainterp_sd = None
+    optheap = None
+
+    def __init__(self, dct={}, cpu=None):
+        self.dct = dct
+        self.constant_classes = {}
+        self.cpu = cpu
+
+    def getptrinfo(self, arg):
+        return self.dct.get(arg, None)
+
+    def make_constant_class(self, arg, cls):
+        self.constant_classes[arg] = cls
+
+class FakeClass(object):
+    pass
+
+class FakeStorage(object):
+    def __init__(self, numb):
+        self.rd_numb = numb
+
+def test_known_classes():
+    box1 = InputArgRef()
+    box2 = InputArgRef()
+    box3 = InputArgRef()
+
+    cls = FakeClass()
+    dct = {box1: InstancePtrInfo(known_class=cls)}
+    optimizer = FakeOptimizer(dct)
+
+    numb_state = NumberingState(4)
+    numb_state.append_int(1) # size of resume block
+    liveboxes = [InputArgInt(), box2, box1, box3]
+
+    serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
+
+    assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0]
+
+    rbox1 = InputArgRef()
+    rbox2 = InputArgRef()
+    rbox3 = InputArgRef()
+    after_optimizer = FakeOptimizer(cpu=FakeCPU({rbox1: cls}))
+    deserialize_optimizer_knowledge(
+        after_optimizer, FakeStorage(numb_state.create_numbering()),
+        [InputArgInt(), rbox2, rbox1, rbox3], liveboxes)
+    assert box1 in after_optimizer.constant_classes
+    assert box2 not in after_optimizer.constant_classes
+    assert box3 not in after_optimizer.constant_classes
+
+
+box_strategy = strategies.builds(InputArgInt) | strategies.builds(InputArgRef)
+tuples = strategies.tuples(box_strategy, strategies.booleans()).filter(
+        lambda (box, known_class): isinstance(box, InputArgRef) or not known_class)
+boxes_known_classes = strategies.lists(tuples, min_size=1)
+
+ at given(boxes_known_classes)
+def test_random_class_knowledge(boxes_known_classes):
+    cls = FakeClass()
+    dct1 = {box: InstancePtrInfo(known_class=cls)
+              for box, known_class in boxes_known_classes
+                  if known_class}
+    optimizer = FakeOptimizer(dct1)
+
+    refboxes = [box for (box, _) in boxes_known_classes
+                    if isinstance(box, InputArgRef)]
+
+    numb_state = NumberingState(1)
+    numb_state.append_int(1) # size of resume block
+    liveboxes = [box for (box, _) in boxes_known_classes]
+
+    serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
+
+    assert len(numb_state.create_numbering().code) == 2 + math.ceil(len(refboxes) / 6.0)
+
+    dct = {box: cls
+              for box, known_class in boxes_known_classes
+                  if known_class}
+    after_optimizer = FakeOptimizer(cpu=FakeCPU(dct))
+    deserialize_optimizer_knowledge(
+        after_optimizer, FakeStorage(numb_state.create_numbering()),
+        liveboxes, liveboxes)
+    for box, known_class in boxes_known_classes:
+        assert (box in after_optimizer.constant_classes) == known_class
+
+class TestOptBridge(LLJitMixin):
+    # integration tests
+    def test_bridge_guard_class(self):
+        myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a'])
+        class A(object):
+            def f(self):
+                return 1
+        class B(A):
+            def f(self):
+                return 2
+        def f(x, y, n):
+            if x:
+                a = A()
+            else:
+                a = B()
+            a.x = 0
+            res = 0
+            while y > 0:
+                myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a)
+                res += a.f()
+                a.x += 1
+                if y > n:
+                    res += 1
+                res += a.f()
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 32, 16])
+        assert res == f(6, 32, 16)
+        self.check_trace_count(3)
+        self.check_resops(guard_class=1)
+
+    def test_bridge_field_read(self):
+        myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a'])
+        class A(object):
+            def f(self):
+                return 1
+        class B(A):
+            def f(self):
+                return 2
+        class M(object):
+            _immutable_fields_ = ['x']
+            def __init__(self, x):
+                self.x = x
+
+        m1 = M(1)
+        m2 = M(2)
+        def f(x, y, n):
+            if x:
+                a = A()
+                a.m = m1
+                a.n = n
+            else:
+                a = B()
+                a.m = m2
+                a.n = n
+            a.x = 0
+            res = 0
+            while y > 0:
+                myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a)
+                n1 = a.n
+                m = jit.promote(a.m)
+                res += m.x
+                a.x += 1
+                if y > n:
+                    res += 1
+                m = jit.promote(a.m)
+                res += m.x
+                res += n1 + a.n
+                y -= 1
+            return res
+        res = self.meta_interp(f, [6, 32, 16])
+        assert res == f(6, 32, 16)
+        self.check_trace_count(3)
+        self.check_resops(guard_value=1)
+        self.check_resops(getfield_gc_i=4) # 3x a.x, 1x a.n
+        self.check_resops(getfield_gc_r=1) # in main loop
+
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -79,6 +79,7 @@
 def test_compile_loop():
     cpu = FakeCPU()
     staticdata = FakeMetaInterpStaticData()
+    staticdata.all_descrs = LLtypeMixin.cpu.setup_descrs()
     staticdata.cpu = cpu
     staticdata.jitlog = jl.JitLogger(cpu)
     staticdata.jitlog.trace_id = 1
diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py
--- a/rpython/jit/metainterp/test/test_greenfield.py
+++ b/rpython/jit/metainterp/test/test_greenfield.py
@@ -49,7 +49,7 @@
         #
         res = self.meta_interp(g, [7])
         assert res == -22
-        self.check_trace_count(6)
+        self.check_trace_count(4)
         self.check_resops(guard_value=0)
 
     def test_green_field_3(self):
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -36,10 +36,12 @@
     rd_consts = []
     rd_virtuals = None
     rd_pendingfields = None
-    rd_count = 0
 
 
 class FakeOptimizer(object):
+    metainterp_sd = None
+    optheap = None
+
     def __init__(self, trace=None):
         self.trace = trace
 
@@ -251,18 +253,17 @@
     def get_current_position_info(self):
         class MyInfo:
             @staticmethod
-            def enumerate_vars(callback_i, callback_r, callback_f, _, index):
+            def enumerate_vars(callback_i, callback_r, callback_f, _):
                 count_i = count_r = count_f = 0
                 for ARG in self.ARGS:
                     if ARG == lltype.Signed:
-                        index = callback_i(index, count_i); count_i += 1
+                        callback_i(count_i); count_i += 1
                     elif ARG == llmemory.GCREF:
-                        index = callback_r(index, count_r); count_r += 1
+                        callback_r(count_r); count_r += 1
                     elif ARG == longlong.FLOATSTORAGE:
-                        index = callback_f(index, count_f); count_f += 1
+                        callback_f(count_f); count_f += 1
                     else:
                         assert 0
-                return index
         return MyInfo()
 
     def setarg_i(self, index, value):
@@ -289,7 +290,8 @@
     assert bh.written_f == expected_f
 
 
-Numbering = create_numbering
+def Numbering(l):
+    return create_numbering([len(l)] + l) # prefix index to the end of thing
 
 def tagconst(i):
     return tag(i + TAG_CONST_OFFSET, TAGCONST)
@@ -299,12 +301,11 @@
     c1, c2, c3 = [ConstInt(111), ConstInt(222), ConstInt(333)]
     storage = Storage()
     storage.rd_consts = [c1, c2, c3]
-    numb = Numbering([tag(0, TAGBOX), tagconst(0),
+    numb = Numbering([3, tag(0, TAGBOX), tagconst(0),
                        NULLREF, tag(0, TAGBOX), tag(1, TAGBOX)] +
-                       [tagconst(1), tagconst(2)] + 
+                       [tagconst(1), tagconst(2)] +
                        [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)])
     storage.rd_numb = numb
-    storage.rd_count = 3
     #
     cpu = MyCPU([42, gcref1, -66])
     metainterp = MyMetaInterp(cpu)
@@ -345,7 +346,7 @@
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
-    numb = Numbering([tag(100, TAGINT)])
+    numb = Numbering([1, tag(100, TAGINT)])
     storage.rd_numb = numb
     #
     cpu = MyCPU([])
@@ -362,10 +363,9 @@
             return s
     class FakeStorage(object):
         rd_virtuals = [FakeVinfo(), None]
-        rd_numb = []
+        rd_numb = Numbering([1])
         rd_consts = []
         rd_pendingfields = None
-        rd_count = 0
     class FakeMetainterp(object):
         _already_allocated_resume_virtuals = None
         cpu = None
@@ -773,12 +773,12 @@
     assert untag(tagged) == (44, TAGINT)
     tagged = memo.getconst(ConstInt(-3))
     assert untag(tagged) == (-3, TAGINT)
-    const = ConstInt(50000)
+    const = ConstInt(5000000)
     tagged = memo.getconst(const)
     index, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert memo.consts[index - TAG_CONST_OFFSET] is const
-    tagged = memo.getconst(ConstInt(50000))
+    tagged = memo.getconst(ConstInt(5000000))
     index2, tagbits = untag(tagged)
     assert tagbits == TAGCONST
     assert index2 == index
@@ -858,7 +858,7 @@
     base = [0, 0, tag(0, TAGBOX), tag(1, TAGINT),
             tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)]
 
-    assert unpack_numbering(numb) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                       tag(0, TAGBOX), tag(1, TAGINT)]
     t.append(0)
     snap2 = t.create_top_snapshot(FakeJitCode("jitcode", 0), 2, Frame(env2),
@@ -872,7 +872,7 @@
     assert numb_state2.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b3: tag(2, TAGBOX)}
     assert numb_state2.liveboxes is not numb_state.liveboxes
-    assert unpack_numbering(numb2) == [0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
+    assert unpack_numbering(numb2) == [17, 0, 0, 0] + base + [0, 2, tag(3, TAGINT), tag(2, TAGBOX),
                                        tag(0, TAGBOX), tag(3, TAGINT)]
 
     t.append(0)
@@ -894,7 +894,7 @@
     assert numb_state3.num_virtuals == 0
     
     assert numb_state3.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)}
-    assert unpack_numbering(numb3) == ([0, 2, tag(3, TAGINT), tag(4, TAGINT),
+    assert unpack_numbering(numb3) == ([17, 0, 0, 2, tag(3, TAGINT), tag(4, TAGINT),
                                        tag(0, TAGBOX), tag(3, TAGINT)] +
                                        base + [0, 2])
 
@@ -911,7 +911,7 @@
     
     assert numb_state4.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL)}
-    assert unpack_numbering(numb4) == [0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
+    assert unpack_numbering(numb4) == [17, 0, 0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL),
                                        tag(0, TAGBOX), tag(3, TAGINT)] + base + [0, 2]
 
     t.append(0)
@@ -930,7 +930,7 @@
 
     assert numb_state5.liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX),
                                      b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)}
-    assert unpack_numbering(numb5) == [
+    assert unpack_numbering(numb5) == [22, 0,
         3, tag(0, TAGBOX), tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL),
         0] + base + [
         2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT)
@@ -949,15 +949,17 @@
     numb_state = memo.number(FakeOptimizer(), 0, i)
     numb = numb_state.create_numbering()
     l = unpack_numbering(numb)
-    assert l[0] == 0
+    assert l[0] == len(l)
+    assert l[1] == 0
     assert l[1] == 0
     assert l[2] == 0
     assert l[3] == 0
+    assert l[4] == 0
     mapping = dict(zip(inpargs, i.inputargs))
     for i, item in enumerate(lst):
-        v, tag = untag(l[i + 4])
+        v, tag = untag(l[i + 6])
         if tag == TAGBOX:
-            assert l[i + 4] == numb_state.liveboxes[mapping[item]]
+            assert l[i + 6] == numb_state.liveboxes[mapping[item]]
         elif tag == TAGCONST:
             assert memo.consts[v].getint() == item.getint()
         elif tag == TAGINT:
@@ -1069,15 +1071,15 @@
     cpu = MyCPU([])
     reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe")
     reader.consume_vref_and_vable(None, None, None)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, sys.maxint, 1, sys.maxint, 2**16)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, 2, 3)
-    reader.cur_index += 2 # framestack
+    reader.resumecodereader.jump(2) # framestack
     _next_section(reader, sys.maxint, 2**16, -65)
 
 def test_virtual_adder_memo_const_sharing():
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**23), ConstInt(-65)]
     storage, t = make_storage(b1s, b2s, b3s)
     metainterp_sd = FakeMetaInterpStaticData()
     memo = ResumeDataLoopMemo(metainterp_sd)
@@ -1087,7 +1089,7 @@
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
-    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
+    b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**24), ConstInt(-65)]
     storage2, t = make_storage(b1s, b2s, b3s)
     i = t.get_iter()
     modifier2 = ResumeDataVirtualAdder(FakeOptimizer(i), storage2, storage2,
@@ -1112,9 +1114,10 @@
                 return True
         class MyInfo:
             @staticmethod
-            def enumerate_vars(callback_i, callback_r, callback_f, _, index):
-                while index < len(self.numb.code):
-                    tagged, _ = resumecode.numb_next_item(self.numb, index)
+            def enumerate_vars(callback_i, callback_r, callback_f, _):
+                index = 0
+                while not self.done_reading():
+                    tagged = self.resumecodereader.peek()
                     _, tag = untag(tagged)
                     if tag == TAGVIRTUAL:
                         kind = REF
@@ -1122,20 +1125,21 @@
                         kind = Whatever()
                     box = self.decode_box(tagged, kind)
                     if box.type == INT:
-                        index = callback_i(index, index)
+                        callback_i(index)
                     elif box.type == REF:
-                        index = callback_r(index, index)
+                        callback_r(index)
                     elif box.type == FLOAT:
-                        index = callback_f(index, index)
+                        callback_f(index)
                     else:
                         assert 0
+                    index += 1
 
-        size, self.cur_index = resumecode.numb_next_item(self.numb, 0)
+        size = self.resumecodereader.next_item()
         assert size == 0
-        size, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
+        size = self.resumecodereader.next_item()
         assert size == 0
-        pc, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
-        jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index)
+        pc = self.resumecodereader.next_item()
+        jitcode_pos = self.resumecodereader.next_item()
 
         self._prepare_next_section(MyInfo())
         return self.lst
@@ -1228,7 +1232,7 @@
     liveboxes = []
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     b3t, b5t = [IntFrontendOp(0), RefFrontendOp(0)]
     b5t.setref_base(demo55o)
@@ -1299,7 +1303,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     b1t, b3t, b4t = [IntFrontendOp(0), IntFrontendOp(0), IntFrontendOp(0)]
     b1t.setint(11)
@@ -1352,7 +1356,7 @@
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     b4t = RefFrontendOp(0)
     newboxes = _resume_remap(liveboxes, [#b2s -- virtual
                                          b4s], b4t)
@@ -1398,7 +1402,7 @@
     modifier._add_pending_fields(FakeOptimizer(), [
         ResOperation(rop.SETFIELD_GC, [b2s, b4s], descr=LLtypeMixin.nextdescr)])
     storage.rd_consts = memo.consts[:]
-    storage.rd_numb = None
+    storage.rd_numb = Numbering([0])
     # resume
     demo55.next = lltype.nullptr(LLtypeMixin.NODE)
     b2t = RefFrontendOp(0)
diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py
--- a/rpython/jit/metainterp/test/test_resumecode.py
+++ b/rpython/jit/metainterp/test/test_resumecode.py
@@ -1,29 +1,62 @@
-
-from rpython.jit.metainterp.resumecode import NUMBERING, NULL_NUMBER
 from rpython.jit.metainterp.resumecode import create_numbering,\
-    unpack_numbering
+    unpack_numbering, Reader, Writer
 from rpython.rtyper.lltypesystem import lltype
 
-from hypothesis import strategies, given
+from hypothesis import strategies, given, example
 
+examples = [
+    [1, 2, 3, 4, 257, 10000, 13, 15],
+    [1, 2, 3, 4],
+    range(1, 10, 2),
+    [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
+]
 
-def test_pack_unpack():
-    examples = [
-        [1, 2, 3, 4, 257, 10000, 13, 15],
-        [1, 2, 3, 4],
-        range(1, 10, 2),
-        [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000]
-    ]
-    for l in examples:
-        n = create_numbering(l)
-        assert unpack_numbering(n) == l
+def hypothesis_and_examples(func):
+    func = given(strategies.lists(strategies.integers(-2**15, 2**15-1)))(func)
+    for ex in examples:
+        func = example(ex)(func)
+    return func
 
- at given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+ at hypothesis_and_examples
 def test_roundtrip(l):
     n = create_numbering(l)
     assert unpack_numbering(n) == l
 
- at given(strategies.lists(strategies.integers(-2**15, 2**15-1)))
+ at hypothesis_and_examples
 def test_compressing(l):
     n = create_numbering(l)
     assert len(n.code) <= len(l) * 3
+
+ at hypothesis_and_examples
+def test_reader(l):
+    n = create_numbering(l)
+    r = Reader(n)
+    for i, elt in enumerate(l):
+        assert r.items_read == i
+        item = r.next_item()
+        assert elt == item
+
+ at hypothesis_and_examples
+def test_writer(l):
+    for size in [len(l), 0]:
+        w = Writer(len(l))
+        for num in l:
+            w.append_int(num)
+        n = w.create_numbering()
+        assert unpack_numbering(n) == l
+
+ at hypothesis_and_examples
+def test_patch(l):
+    for middle in range(len(l)):
+        l1 = l[:middle]
+        l2 = l[middle:]
+        w = Writer(len(l))
+        w.append_int(0)
+        for num in l1:
+            w.append_int(num)
+        w.patch_current_size(0)
+        for num in l2:
+            w.append_int(num)
+        n = w.create_numbering()
+        assert unpack_numbering(n)[1:] == l
+        assert unpack_numbering(n)[0] == middle + 1
diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -2,7 +2,6 @@
 from rpython.jit.metainterp import history
 from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem
 from rpython.jit.metainterp.warmstate import wrap, unwrap
-from rpython.jit.metainterp.resumecode import numb_next_item
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import rvirtualizable
 from rpython.rtyper.lltypesystem import lltype, llmemory
@@ -127,24 +126,21 @@
                 size += 1
             return size
 
-        def write_from_resume_data_partial(virtualizable, reader, index, numb):
+        def write_from_resume_data_partial(virtualizable, reader):
             virtualizable = cast_gcref_to_vtype(virtualizable)
             # Load values from the reader (see resume.py) described by
             # the list of numbers 'nums', and write them in their proper
             # place in the 'virtualizable'.
             for FIELDTYPE, fieldname in unroll_static_fields:
-                item, index = numb_next_item(numb, index)
-                x = reader.load_value_of_type(FIELDTYPE, item)
+                x = reader.load_next_value_of_type(FIELDTYPE)
                 setattr(virtualizable, fieldname, x)
             for ARRAYITEMTYPE, fieldname in unroll_array_fields:
                 lst = getattr(virtualizable, fieldname)
                 for j in range(getlength(lst)):
-                    item, index = numb_next_item(numb, index)                    
-                    x = reader.load_value_of_type(ARRAYITEMTYPE, item)
+                    x = reader.load_next_value_of_type(ARRAYITEMTYPE)
                     setarrayitem(lst, j, x)
-            return index
 
-        def load_list_of_boxes(virtualizable, reader, vable_box, numb, index):
+        def load_list_of_boxes(virtualizable, reader, vable_box):
             virtualizable = cast_gcref_to_vtype(virtualizable)
             # Uses 'virtualizable' only to know the length of the arrays;
             # does not write anything into it.  The returned list is in
@@ -152,17 +148,15 @@
             # the virtualizable itself.
             boxes = []
             for FIELDTYPE, fieldname in unroll_static_fields:
-                item, index = numb_next_item(numb, index)
-                box = reader.decode_box_of_type(FIELDTYPE, item)
+                box = reader.next_box_of_type(FIELDTYPE)
                 boxes.append(box)
             for ARRAYITEMTYPE, fieldname in unroll_array_fields:
                 lst = getattr(virtualizable, fieldname)
                 for j in range(getlength(lst)):
-                    item, index = numb_next_item(numb, index)                    
-                    box = reader.decode_box_of_type(ARRAYITEMTYPE, item)
+                    box = reader.next_box_of_type(ARRAYITEMTYPE)
                     boxes.append(box)
             boxes.append(vable_box)
-            return boxes, index
+            return boxes
 
         def check_boxes(virtualizable, boxes):
             virtualizable = cast_gcref_to_vtype(virtualizable)

From pypy.commits at gmail.com  Sun Feb 26 13:34:37 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 26 Feb 2017 10:34:37 -0800 (PST)
Subject: [pypy-commit] pypy default: an FAQ entry about PyPy2
Message-ID: <58b31fbd.8d202e0a.b64b.12c9@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r90368:f7f4d72c69da
Date: 2017-02-26 19:33 +0100
http://bitbucket.org/pypy/pypy/changeset/f7f4d72c69da/

Log:	an FAQ entry about PyPy2

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -437,3 +437,11 @@
 but so far there doesn't seem to be an overwhelming commercial interest in it.
 
 .. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation
+
+
+How long will PyPy support Python2?
+-----------------------------------
+
+Since RPython is built on top of Python2 and that is extremely unlikely to
+change, the Python2 version of PyPy will be around "forever", i.e. as long as
+PyPy itself is around.

From pypy.commits at gmail.com  Sun Feb 26 17:42:13 2017
From: pypy.commits at gmail.com (fijal)
Date: Sun, 26 Feb 2017 14:42:13 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: * whack whack whack
Message-ID: <58b359c5.43ca190a.c250d.18c4@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90369:50071ee2bad7
Date: 2017-02-26 20:07 +0100
http://bitbucket.org/pypy/pypy/changeset/50071ee2bad7/

Log:	* whack whack whack
	* have a more sensible 'check for utf8' and a slower version if we
	pass something awkward as errorhandler

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1679,6 +1679,13 @@
     def utf8_w(self, w_obj):
         return w_obj.utf8_w(self)
 
+    def unicode_w(self, w_obj):
+        return self.utf8_w(w_obj).decode('utf8')
+
+    def newunicode(self, u):
+        assert isinstance(u, unicode)
+        return self.newutf8(u.encode("utf8"), len(u))
+
     def convert_to_w_unicode(self, w_obj):
         return w_obj.convert_to_w_unicode(self)
 
@@ -1693,13 +1700,12 @@
                         "characters")
         return rstring.assert_str0(result)
 
-    def realunicode_w(self, w_obj):
-        # Like unicode_w, but only works if w_obj is really of type
+    def realutf8_w(self, w_obj):
+        # Like utf8_w, but only works if w_obj is really of type
         # 'unicode'.
-        xxx
         if not self.isinstance_w(w_obj, self.w_unicode):
             raise oefmt(self.w_TypeError, "argument must be a unicode")
-        return self.unicode_w(w_obj)
+        return self.utf8_w(w_obj)
 
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -59,10 +59,12 @@
     # If there happen to be two 3-bytes encoding a pair of surrogates,
     # you still get two surrogate unicode characters in the result.
     # These are the Python2 rules; Python3 differs.
-    consumed, length = rutf8.str_check_utf8(
-        string, len(string), "strict", final=True,
-        errorhandler=decode_error_handler(space),
-        allow_surrogates=True)
+    try:
+        consumed, length = rutf8.str_check_utf8(string, len(string), True)
+    except rutf8.Utf8CheckError as e:
+        decode_error_handler(space)('strict', 'utf8', e.msg, string, e.startpos,
+                                    e.endpos)
+        raise False, "unreachable"
     return length
 
 def encode_utf8(space, uni):
@@ -78,4 +80,69 @@
 def utf8_encode_ascii(utf8, utf8len, errors, errorhandler):
     if len(utf8) == utf8len:
         return utf8
-    xxx
+    return rutf8.utf8_encode_ascii(utf8, errors, 'ascii',
+                                   'ordinal not in range (128)',
+                                   errorhandler)
+
+def str_decode_ascii(s, slen, errors, final, errorhandler):
+    try:
+        rutf8.check_ascii(s)
+        return s
+    except rutf8.AsciiCheckError:
+        return rutf8.str_decode_ascii(s, errors, errorhandler)
+
+# XXX wrappers, think about speed
+
+class DecodeWrapper(object):
+    def __init__(self, handler):
+        self.orig = handler
+
+    def handle(self, errors, encoding, msg, s, pos, endpos):
+        s, p, lgt = self.orig(errors, encoding, msg, s, pos, endpos)
+        return s.decode("utf8"), p
+
+class EncodeWrapper(object):
+    def __init__(self, handler):
+        self.orig = handler
+
+    def handle(self, errors, encoding, msg, s, pos, endpos):
+        s, rs, p, lgt = self.orig(errors, encoding, msg, s.encode("utf8"), pos, endpos)
+        return s, rs, p
+
+def utf8_encode_utf_7(utf8, utf8len, errors, errorhandler):
+    u = utf8.decode("utf8")
+    w = EncodeWrapper(errorhandler)
+    return runicode.unicode_encode_utf_7(u, len(u), errors, w.handle)
+
+def str_decode_utf_7(string, lgt, errors, final, errorhandler):
+    w = DecodeWrapper(errorhandler)
+    u, pos = runicode.str_decode_utf_7(string, lgt, errors, final, w.handle)
+    return u.encode('utf8'), pos, len(u)
+
+def str_decode_unicode_escape(s, slen, errors, final, errorhandler, ud_handler):
+    w = DecodeWrapper(errorhandler)
+    u, pos = runicode.str_decode_unicode_escape(s, slen, errors, final, w.handle,
+                                                ud_handler)
+    return u.encode('utf8'), pos, len(u)
+
+def str_decode_raw_unicode_escape(s, slen, errors, final, errorhandler):
+    w = DecodeWrapper(errorhandler)
+    u, pos = runicode.str_decode_raw_unicode_escape(s, slen, errors, final,
+                                                    w.handle)
+    return u.encode('utf8'), pos, len(u)
+
+def str_decode_utf8(s, slen, errors, final, errorhandler):
+    w = DecodeWrapper(errorhandler)
+    u, pos = runicode.str_decode_utf_8_impl(s, slen, errors, final, w.handle,
+        runicode.allow_surrogate_by_default)
+    return u.encode('utf8'), pos, len(u)
+
+def utf8_encode_utf_16(utf8, utf8len, errors, errorhandler):
+    w = EncodeWrapper(errorhandler)
+    u = utf8.decode("utf8")
+    return runicode.unicode_encode_utf_16(u, len(u), errors, w.handle)
+
+def utf8_encode_latin_1(utf8, utf8len, errors, errorhandler):
+    w = EncodeWrapper(errorhandler)
+    u = utf8.decode("utf8")
+    return runicode.unicode_encode_latin_1(u, len(u), errors, w.handle)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -1,4 +1,4 @@
-from rpython.rlib import jit
+from rpython.rlib import jit, rutf8
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rstring import UnicodeBuilder
 from rpython.rlib.runicode import code_to_unichr, MAXUNICODE
@@ -39,7 +39,7 @@
                 w_input = space.newbytes(input)
             else:
                 w_cls = space.w_UnicodeEncodeError
-                w_input = space.newunicode(input)
+                w_input = space.newutf8(input, -1)
             w_exc =  space.call_function(
                 w_cls,
                 space.newtext(encoding),
@@ -65,8 +65,8 @@
                 raise oefmt(space.w_IndexError,
                             "position %d from error handler out of bounds",
                             newpos)
-            replace = space.unicode_w(w_replace)
-            return replace, newpos
+            w_replace = space.convert_to_w_unicode(w_replace)
+            return w_replace._utf8, newpos, w_replace._length
         return call_errorhandler
 
     def make_decode_errorhandler(self, space):
@@ -76,9 +76,9 @@
         errorhandler = self._make_errorhandler(space, False)
         def encode_call_errorhandler(errors, encoding, reason, input, startpos,
                                      endpos):
-            replace, newpos = errorhandler(errors, encoding, reason, input,
+            replace, newpos, lgt = errorhandler(errors, encoding, reason, input,
                                            startpos, endpos)
-            return replace, None, newpos
+            return replace, None, newpos, lgt
         return encode_call_errorhandler
 
     def get_unicodedata_handler(self, space):
@@ -190,7 +190,9 @@
 def ignore_errors(space, w_exc):
     check_exception(space, w_exc)
     w_end = space.getattr(w_exc, space.newtext('end'))
-    return space.newtuple([space.newunicode(u''), w_end])
+    return space.newtuple([space.newutf8('', 0), w_end])
+
+REPLACEMENT = u'\ufffd'.encode('utf8')
 
 def replace_errors(space, w_exc):
     check_exception(space, w_exc)
@@ -198,14 +200,14 @@
     w_end = space.getattr(w_exc, space.newtext('end'))
     size = space.int_w(w_end) - space.int_w(w_start)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
-        text = u'?' * size
-        return space.newtuple([space.newunicode(text), w_end])
+        text = '?' * size
+        return space.newtuple([space.newutf8(text, size), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
-        text = u'\ufffd'
-        return space.newtuple([space.newunicode(text), w_end])
+        text = REPLACEMENT
+        return space.newtuple([space.newutf8(text, 1), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeTranslateError):
-        text = u'\ufffd' * size
-        return space.newtuple([space.newunicode(text), w_end])
+        text = REPLACEMENT * size
+        return space.newtuple([space.newutf8(text, size), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -392,14 +394,17 @@
     @unwrap_spec(string='bufferstr', errors='str_or_None',
                  w_final=WrappedDefault(False))
     def wrap_decoder(space, string, errors="strict", w_final=None):
+        from pypy.interpreter import unicodehelper
+
         if errors is None:
             errors = 'strict'
         final = space.is_true(w_final)
         state = space.fromcache(CodecState)
-        func = getattr(runicode, rname)
-        result, consumed = func(string, len(string), errors,
+        func = getattr(unicodehelper, rname)
+        result, consumed, length = func(string, len(string), errors,
                                 final, state.decode_error_handler)
-        return space.newtuple([space.newunicode(result), space.newint(consumed)])
+        return space.newtuple([space.newutf8(result, length),
+                               space.newint(consumed)])
     wrap_decoder.func_name = rname
     globals()[name] = wrap_decoder
 
@@ -441,33 +446,42 @@
 # "allow_surrogates=True"
 @unwrap_spec(utf8='utf8', errors='str_or_None')
 def utf_8_encode(space, utf8, utf8len, errors="strict"):
-    if errors is None:
-        errors = 'strict'
-    xxx
-    state = space.fromcache(CodecState)
-    # NB. can't call unicode_encode_utf_8() directly because that's
-    # an @elidable function nowadays.  Instead, we need the _impl().
-    # (The problem is the errorhandler, which calls arbitrary Python.)
-    result = runicode.unicode_encode_utf_8_impl(
-        uni, len(uni), errors, state.encode_error_handler,
-        allow_surrogates=True)
-    return space.newtuple([space.newbytes(result), space.newint(len(uni))])
+    return space.newtuple([space.newbytes(utf8), space.newint(utf8len)])
 
 @unwrap_spec(string='bufferstr', errors='str_or_None',
              w_final = WrappedDefault(False))
 def utf_8_decode(space, string, errors="strict", w_final=None):
+    from pypy.interpreter import unicodehelper
+
     if errors is None:
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    # NB. can't call str_decode_utf_8() directly because that's
-    # an @elidable function nowadays.  Instead, we need the _impl().
-    # (The problem is the errorhandler, which calls arbitrary Python.)
-    result, consumed = runicode.str_decode_utf_8_impl(
-        string, len(string), errors,
-        final, state.decode_error_handler,
-        allow_surrogates=True)
-    return space.newtuple([space.newunicode(result), space.newint(consumed)])
+    # call the fast version for checking
+    try:
+        consumed, lgt = rutf8.str_check_utf8(string, len(string), final)
+    except rutf8.Utf8CheckError as e:
+        if errors == 'strict':
+            # just raise
+            state.decode_error_handler(errors, 'utf8', e.msg, string,
+                                       e.startpos, e.endpos)
+            assert False, "raises"
+        # XXX do the way aroun runicode - we can optimize it later if we
+        # decide we care about obscure cases
+        res, consumed, lgt = unicodehelper.str_decode_utf8(string, len(string),
+            errors, final, state.decode_error_handler)
+        return space.newtuple([space.newutf8(res, lgt),
+                           space.newint(consumed)])
+    #result, consumed = runicode.str_decode_utf_8_impl(
+    #    string, len(string), errors,
+    #    final, state.decode_error_handler,
+    #    allow_surrogates=True)
+    if final or consumed == len(string):
+        return space.newtuple([space.newutf8(string, lgt),
+                               space.newint(consumed)])
+
+    return space.newtuple([space.newutf8(string[:consumed], lgt),
+                           space.newint(consumed)])
 
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
@@ -655,6 +669,8 @@
 @unwrap_spec(string='bufferstr', errors='str_or_None',
              w_final=WrappedDefault(False))
 def unicode_escape_decode(space, string, errors="strict", w_final=None):
+    from pypy.interpreter import unicodehelper
+
     if errors is None:
         errors = 'strict'
     final = space.is_true(w_final)
@@ -662,12 +678,12 @@
 
     unicode_name_handler = state.get_unicodedata_handler(space)
 
-    result, consumed = runicode.str_decode_unicode_escape(
+    result, consumed, lgt = unicodehelper.str_decode_unicode_escape(
         string, len(string), errors,
         final, state.decode_error_handler,
         unicode_name_handler)
 
-    return space.newtuple([space.newunicode(result), space.newint(consumed)])
+    return space.newtuple([space.newutf8(result, lgt), space.newint(consumed)])
 
 # ____________________________________________________________
 # Unicode-internal
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -719,7 +719,7 @@
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
         space.realstr_w(w_encoding)
-        space.realunicode_w(w_object)
+        space.realutf8_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
         space.realstr_w(w_reason)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -460,6 +460,11 @@
     def str_w(self, space):
         return self._value
 
+    def utf8_w(self, space):
+        # Use the default encoding.                                             
+        encoding = getdefaultencoding(space)
+        return space.utf8_w(decode_object(space, self, encoding, None))
+
     def buffer_w(self, space, flags):
         space.check_buf_flags(flags, True)
         return StringBuffer(self._value)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -650,9 +650,9 @@
         if type(w_key) is self.space.StringObjectCls:
             self.switch_to_bytes_strategy(w_dict)
             return
-        elif type(w_key) is self.space.UnicodeObjectCls:
-            self.switch_to_unicode_strategy(w_dict)
-            return
+        #elif type(w_key) is self.space.UnicodeObjectCls:
+        #    self.switch_to_unicode_strategy(w_dict)
+        #    return
         w_type = self.space.type(w_key)
         if self.space.is_w(w_type, self.space.w_int):
             self.switch_to_int_strategy(w_dict)
@@ -668,6 +668,7 @@
         w_dict.dstorage = storage
 
     def switch_to_unicode_strategy(self, w_dict):
+        xxx
         strategy = self.space.fromcache(UnicodeDictStrategy)
         storage = strategy.get_empty_storage()
         w_dict.set_strategy(strategy)
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -511,7 +511,8 @@
             pass
         else:
             return space.newbytes(result)
-    fmt = space.unicode_w(w_fmt)
+    # XXX for now, this is performance critical
+    fmt = space.utf8_w(w_fmt).decode("utf8")
     formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict)
     result = formatter.format()
     return space.newunicode(result)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -491,8 +491,8 @@
             return w_obj.listview_unicode()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_unicode()
-        if isinstance(w_obj, W_UnicodeObject) and self._uni_uses_no_iter(w_obj):
-            return w_obj.listview_unicode()
+        #if isinstance(w_obj, W_UnicodeObject) and self._uni_uses_no_iter(w_obj):
+        #    return w_obj.listview_unicode()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_unicode()
         return None
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -799,8 +799,8 @@
             strategy = self.space.fromcache(IntegerSetStrategy)
         elif type(w_key) is W_BytesObject:
             strategy = self.space.fromcache(BytesSetStrategy)
-        elif type(w_key) is W_UnicodeObject:
-            strategy = self.space.fromcache(UnicodeSetStrategy)
+        #elif type(w_key) is W_UnicodeObject:
+        #    strategy = self.space.fromcache(UnicodeSetStrategy)
         elif self.space.type(w_key).compares_by_identity():
             strategy = self.space.fromcache(IdentitySetStrategy)
         else:
@@ -1640,13 +1640,13 @@
         return
 
     # check for unicode
-    for w_item in iterable_w:
-        if type(w_item) is not W_UnicodeObject:
-            break
-    else:
-        w_set.strategy = space.fromcache(UnicodeSetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
-        return
+    #for w_item in iterable_w:
+    #    if type(w_item) is not W_UnicodeObject:
+    #        break
+    #else:
+    #    w_set.strategy = space.fromcache(UnicodeSetStrategy)
+    #    w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+    #    return
 
     # check for compares by identity
     for w_item in iterable_w:
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -142,6 +142,7 @@
         assert self.space.listview_bytes(w_d) == ["a", "b"]
 
     def test_listview_unicode_dict(self):
+        py.test.skip("listview_unicode disabled")
         w = self.space.wrap
         w_d = self.space.newdict()
         w_d.initialize_content([(w(u"a"), w(1)), (w(u"b"), w(2))])
@@ -175,6 +176,7 @@
         # XXX: it would be nice if the test passed without monkeypatch.undo(),
         # but we need space.newlist_unicode for it
         monkeypatch.undo() 
+        py.test.skip("listview_unicode disabled")
         w_d = self.space.newdict()
         w_d.initialize_content([(w(u"a"), w(1)), (w(u"b"), w(6))])
         w_l = self.space.call_method(w_d, "keys")
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -20,8 +20,8 @@
                           IntegerListStrategy)
         assert isinstance(W_ListObject(space, [wb('a'), wb('b')]).strategy,
                           BytesListStrategy)
-        assert isinstance(W_ListObject(space, [w(u'a'), w(u'b')]).strategy,
-                          UnicodeListStrategy)
+        #assert isinstance(W_ListObject(space, [w(u'a'), w(u'b')]).strategy,
+        #                  UnicodeListStrategy)
         assert isinstance(W_ListObject(space, [w(u'a'), wb('b')]).strategy,
                           ObjectListStrategy) # mixed unicode and bytes
 
@@ -47,7 +47,7 @@
         l = W_ListObject(space, [])
         assert isinstance(l.strategy, EmptyListStrategy)
         l.append(w(u'a'))
-        assert isinstance(l.strategy, UnicodeListStrategy)
+        #assert isinstance(l.strategy, UnicodeListStrategy)
 
         l = W_ListObject(space, [])
         assert isinstance(l.strategy, EmptyListStrategy)
@@ -74,6 +74,7 @@
         assert isinstance(l.strategy, ObjectListStrategy)
 
     def test_unicode_to_any(self):
+        py.test.skip("disabled")
         space = self.space
         l = W_ListObject(space, [space.wrap(u'a'), space.wrap(u'b'), space.wrap(u'c')])
         assert isinstance(l.strategy, UnicodeListStrategy)
@@ -117,7 +118,7 @@
 
         # UnicodeStrategy to ObjectStrategy
         l = W_ListObject(space, [w(u'a'),w(u'b'),w(u'c')])
-        assert isinstance(l.strategy, UnicodeListStrategy)
+        #assert isinstance(l.strategy, UnicodeListStrategy)
         l.setitem(0, w(2))
         assert isinstance(l.strategy, ObjectListStrategy)
 
@@ -145,7 +146,7 @@
 
         # UnicodeStrategy
         l = W_ListObject(space, [w(u'a'),w(u'b'),w(u'c')])
-        assert isinstance(l.strategy, UnicodeListStrategy)
+        #assert isinstance(l.strategy, UnicodeListStrategy)
         l.insert(3, w(2))
         assert isinstance(l.strategy, ObjectListStrategy)
 
@@ -225,7 +226,7 @@
 
         # UnicodeStrategy to ObjectStrategy
         l = W_ListObject(space, [w(u'a'), w(u'b'), w(u'c')])
-        assert isinstance(l.strategy, UnicodeListStrategy)
+        #assert isinstance(l.strategy, UnicodeListStrategy)
         l.setslice(0, 1, 2, W_ListObject(space, [w(1), w(2), w(3)]))
         assert isinstance(l.strategy, ObjectListStrategy)
 
@@ -275,7 +276,7 @@
         l = W_ListObject(space, wrapitems([u"a",u"b",u"c",u"d",u"e"]))
         other = W_ListObject(space, wrapitems([u"a", u"b", u"c"]))
         keep_other_strategy(l, 0, 2, other.length(), other)
-        assert l.strategy is space.fromcache(UnicodeListStrategy)
+        #assert l.strategy is space.fromcache(UnicodeListStrategy)
 
         l = W_ListObject(space, wrapitems([1.1, 2.2, 3.3, 4.4, 5.5]))
         other = W_ListObject(space, [])
@@ -345,7 +346,7 @@
         empty = W_ListObject(space, [])
         assert isinstance(empty.strategy, EmptyListStrategy)
         empty.extend(W_ListObject(space, [w(u"a"), w(u"b"), w(u"c")]))
-        assert isinstance(empty.strategy, UnicodeListStrategy)
+        #assert isinstance(empty.strategy, UnicodeListStrategy)
 
         empty = W_ListObject(space, [])
         assert isinstance(empty.strategy, EmptyListStrategy)
@@ -601,7 +602,7 @@
         l1 = W_ListObject(self.space, [self.space.newbytes("eins"), self.space.newbytes("zwei")])
         assert isinstance(l1.strategy, BytesListStrategy)
         l2 = W_ListObject(self.space, [self.space.newunicode(u"eins"), self.space.newunicode(u"zwei")])
-        assert isinstance(l2.strategy, UnicodeListStrategy)
+        #assert isinstance(l2.strategy, UnicodeListStrategy)
         l3 = W_ListObject(self.space, [self.space.newbytes("eins"), self.space.newunicode(u"zwei")])
         assert isinstance(l3.strategy, ObjectListStrategy)
 
@@ -612,6 +613,7 @@
         assert space.listview_bytes(w_l) == ["a", "b"]
 
     def test_listview_unicode(self):
+        py.test.skip("disabled")
         space = self.space
         assert space.listview_unicode(space.wrap(1)) == None
         w_l = self.space.newlist([self.space.wrap(u'a'), self.space.wrap(u'b')])
@@ -624,6 +626,7 @@
         assert space.str_w(space.call_method(space.wrap("c"), "join", w_l)) == "acb"
         #
         # the same for unicode
+        py.test.skip("disabled")
         w_l = self.space.newlist([self.space.wrap(u'a'), self.space.wrap(u'b')])
         w_l.getitems = None
         assert space.unicode_w(space.call_method(space.wrap(u"c"), "join", w_l)) == u"acb"
@@ -636,6 +639,7 @@
         assert space.is_w(space.call_method(space.wrap(" -- "), "join", w_l), w_text)
         #
         # the same for unicode
+        py.test.skip("disabled")
         w_text = space.wrap(u"text")
         w_l = self.space.newlist([w_text])
         w_l.getitems = None
@@ -665,6 +669,7 @@
         assert space.listview_bytes(w_l4) == ["a", "b", "c"]
 
     def test_unicode_uses_newlist_unicode(self):
+        py.test.skip("disabled")
         space = self.space
         w_u = space.wrap(u"a b c")
         space.newlist = None
@@ -720,6 +725,7 @@
         assert self.space.listview_bytes(w_l) == ["a", "b"]
 
     def test_listview_unicode_list(self):
+        py.test.skip("disabled")
         space = self.space
         w_l = W_ListObject(space, [space.wrap(u"a"), space.wrap(u"b")])
         assert self.space.listview_unicode(w_l) == [u"a", u"b"]
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -17,7 +17,7 @@
         cls.w_cpython_apptest = space.wrap(option.runappdirect and not hasattr(sys, 'pypy_translation_info'))
 
         def w_unwrap_wrap_unicode(space, w_obj):
-            return space.wrap(space.unicode_w(w_obj))
+            return space.newutf8(space.utf8_w(w_obj), w_obj._length)
         cls.w_unwrap_wrap_unicode = space.wrap(gateway.interp2app(w_unwrap_wrap_unicode))
         def w_unwrap_wrap_str(space, w_obj):
             return space.wrap(space.str_w(w_obj))
@@ -194,7 +194,8 @@
         assert id('') == (256 << 4) | 11     # always
         assert id(u'') == (257 << 4) | 11
         assert id('a') == (ord('a') << 4) | 11
-        assert id(u'\u1234') == ((~0x1234) << 4) | 11
+        # we no longer cache unicodes <128
+        # assert id(u'\u1234') == ((~0x1234) << 4) | 11
 
     def test_id_of_tuples(self):
         l = []
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -105,8 +105,8 @@
         w_list = self.space.iter(W_ListObject(self.space, [w(u"1"), w(u"2"), w(u"3")]))
         w_set = W_SetObject(self.space)
         _initialize_set(self.space, w_set, w_list)
-        assert w_set.strategy is self.space.fromcache(UnicodeSetStrategy)
-        assert w_set.strategy.unerase(w_set.sstorage) == {u"1":None, u"2":None, u"3":None}
+        #assert w_set.strategy is self.space.fromcache(UnicodeSetStrategy)
+        #assert w_set.strategy.unerase(w_set.sstorage) == {u"1":None, u"2":None, u"3":None}
 
         w_list = W_ListObject(self.space, [w("1"), w(2), w("3")])
         w_set = W_SetObject(self.space)
diff --git a/pypy/objspace/std/test/test_setstrategies.py b/pypy/objspace/std/test/test_setstrategies.py
--- a/pypy/objspace/std/test/test_setstrategies.py
+++ b/pypy/objspace/std/test/test_setstrategies.py
@@ -1,3 +1,5 @@
+
+import py
 from pypy.objspace.std.setobject import W_SetObject
 from pypy.objspace.std.setobject import (
     BytesIteratorImplementation, BytesSetStrategy, EmptySetStrategy,
@@ -26,8 +28,8 @@
         s = W_SetObject(self.space, self.wrapped(["a", "b"]))
         assert s.strategy is self.space.fromcache(BytesSetStrategy)
 
-        s = W_SetObject(self.space, self.wrapped([u"a", u"b"]))
-        assert s.strategy is self.space.fromcache(UnicodeSetStrategy)
+        #s = W_SetObject(self.space, self.wrapped([u"a", u"b"]))
+        #assert s.strategy is self.space.fromcache(UnicodeSetStrategy)
 
     def test_switch_to_object(self):
         s = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
@@ -40,6 +42,7 @@
         assert s1.strategy is self.space.fromcache(ObjectSetStrategy)
 
     def test_switch_to_unicode(self):
+        py.test.skip("disabled")
         s = W_SetObject(self.space, self.wrapped([]))
         s.add(self.space.wrap(u"six"))
         assert s.strategy is self.space.fromcache(UnicodeSetStrategy)
@@ -128,11 +131,11 @@
         assert space.unwrap(it.next()) == "a"
         assert space.unwrap(it.next()) == "b"
         #
-        s = W_SetObject(space, self.wrapped([u"a", u"b"]))
-        it = s.iter()
-        assert isinstance(it, UnicodeIteratorImplementation)
-        assert space.unwrap(it.next()) == u"a"
-        assert space.unwrap(it.next()) == u"b"
+        #s = W_SetObject(space, self.wrapped([u"a", u"b"]))
+        #it = s.iter()
+        #assert isinstance(it, UnicodeIteratorImplementation)
+        #assert space.unwrap(it.next()) == u"a"
+        #assert space.unwrap(it.next()) == u"b"
 
     def test_listview(self):
         space = self.space
@@ -142,5 +145,5 @@
         s = W_SetObject(space, self.wrapped(["a", "b"]))
         assert sorted(space.listview_bytes(s)) == ["a", "b"]
         #
-        s = W_SetObject(space, self.wrapped([u"a", u"b"]))
-        assert sorted(space.listview_unicode(s)) == [u"a", u"b"]
+        #s = W_SetObject(space, self.wrapped([u"a", u"b"]))
+        #assert sorted(space.listview_unicode(s)) == [u"a", u"b"]
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -518,7 +518,7 @@
         raises(TypeError, u'hello'.translate)
         raises(TypeError, u'abababc'.translate, {ord('a'):''})
 
-    def test_unicode_form_encoded_object(self):
+    def test_unicode_from_encoded_object(self):
         assert unicode('x', 'utf-8') == u'x'
         assert unicode('x', 'utf-8', 'strict') == u'x'
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -70,12 +70,14 @@
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        s = space.unicode_w(self)
-        if len(s) > 1:
+        s = space.utf8_w(self)
+        if len(s) > 2:
             uid = compute_unique_id(s)
         else:            # strings of len <= 1 are unique-ified
             if len(s) == 1:
                 base = ~ord(s[0])      # negative base values
+            elif len(s) == 2:
+                base = ~((ord(s[1]) << 8) | ord(s[0]))
             else:
                 base = 257       # empty unicode string: base value 257
             uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL
@@ -88,9 +90,11 @@
         return self._utf8
 
     def readbuf_w(self, space):
+        # XXX for now
         from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
-        builder = StringBuilder(len(self._value) * UNICODE_SIZE)
-        for unich in self._value:
+        v = self._utf8.decode("utf8")
+        builder = StringBuilder(len(v) * UNICODE_SIZE)
+        for unich in v:
             pack_unichar(unich, builder)
         return StringBuffer(builder.build())
 
@@ -331,7 +335,8 @@
         formatter = newformat.unicode_formatter(space, spec)
         self2 = unicode_from_object(space, self)
         assert isinstance(self2, W_UnicodeObject)
-        return formatter.format_string(self2._value)
+        # XXX
+        return formatter.format_string(self2._utf8.decode("utf8"))
 
     def descr_mod(self, space, w_values):
         return mod_format(space, self, w_values, do_unicode=True)
@@ -617,21 +622,28 @@
         if errors is None or errors == 'strict':
             try:
                 if encoding == 'ascii':
-                    u = space.unicode_w(w_object)
-                    eh = unicodehelper.raise_unicode_exception_encode
-                    return space.newbytes(unicode_encode_ascii(
-                            u, len(u), None, errorhandler=eh))
+                    s = space.utf8_w(w_object)
+                    try:
+                        rutf8.check_ascii(s)
+                    except rutf8.AsciiCheckError as a:
+                        eh = unicodehelper.raise_unicode_exception_encode
+                        eh(None, "ascii", "ordinal not in range(128)", s,
+                            a.pos, a.pos + 1)
+                        assert False, "always raises"
+                    return space.newbytes(s)
                 if encoding == 'utf-8':
-                    u = space.unicode_w(w_object)
-                    eh = unicodehelper.raise_unicode_exception_encode
-                    return space.newbytes(unicode_encode_utf_8(
-                            u, len(u), None, errorhandler=eh,
-                            allow_surrogates=True))
+                    u = space.utf8_w(w_object)
+                    return space.newbytes(u)
+                    # XXX is this enough?
+                    #eh = unicodehelper.raise_unicode_exception_encode
+                    #return space.newbytes(unicode_encode_utf_8(
+                    #        u, len(u), None, errorhandler=eh,
+                    #        allow_surrogates=True))
             except unicodehelper.RUnicodeEncodeError as ue:
                 raise OperationError(space.w_UnicodeEncodeError,
                                      space.newtuple([
                     space.newtext(ue.encoding),
-                    space.newunicode(ue.object),
+                    space.newutf8(ue.object, -1),
                     space.newint(ue.start),
                     space.newint(ue.end),
                     space.newtext(ue.reason)]))
@@ -665,13 +677,15 @@
                 assert False
             return space.newutf8(s, len(s))
         if encoding == 'utf-8':
-            yyy
             s = space.charbuf_w(w_obj)
             eh = unicodehelper.decode_error_handler(space)
-            return space.newunicode(str_decode_utf_8(
-                    s, len(s), None, final=True, errorhandler=eh,
-                    allow_surrogates=True)[0])
-    xxx
+            try:
+                _, lgt = rutf8.str_check_utf8(s, len(s), final=True,
+                                              allow_surrogates=True)
+            except rutf8.Utf8CheckError as e:
+                eh(None, 'utf8', e.msg, s, e.startpos, e.endpos)
+                assert False, "has to raise"
+            return space.newutf8(s, lgt)
     w_codecs = space.getbuiltinmodule("_codecs")
     w_decode = space.getattr(w_codecs, space.newtext("decode"))
     if errors is None:
@@ -723,7 +737,6 @@
     # this is a performance and bootstrapping hack
     encoding = getdefaultencoding(space)
     if encoding != 'ascii':
-        xxx
         return unicode_from_encoded_object(space, w_bytes, encoding, "strict")
     s = space.bytes_w(w_bytes)
     try:
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -1,9 +1,6 @@
 
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib import runicode, jit
-from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib.nonconst import NonConstant
-from rpython.tool.sourcetools import func_with_new_name
 
 def unichr_as_utf8(code):
     """ Encode code (numeric value) as utf8 encoded string
@@ -81,6 +78,38 @@
 def default_unicode_error_check(*args):
     xxx
 
+def utf8_encode_ascii(s, errors, encoding, msg, errorhandler):
+    res = StringBuilder(len(s))
+    u_pos = 0
+    pos = 0
+    while pos < len(s):
+        chr1 = s[pos]
+        if ord(chr1) < 0x80:
+            res.append(chr1)
+        else:
+            repl, _, _, _ = errorhandler(errors, encoding, msg, s, u_pos, u_pos + 1)
+            res.append(repl)
+        u_pos += 1
+        pos = next_codepoint_pos(s, pos)
+    return res.build()
+
+def str_decode_ascii(s, errors, errorhandler):
+    # ASCII is equivalent to the first 128 ordinals in Unicode.
+    size = len(s)
+    result = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        c = s[pos]
+        if ord(c) < 128:
+            result.append(c)
+        else:
+            r, _, _ = errorhandler(errors, "ascii", "ordinal not in range(128)",
+                                   s,  pos, pos + 1)
+            result.append(r)
+        pos += 1
+    return result.build(), pos, -1
+
+
 def default_unicode_error_decode(errors, encoding, message, s, pos, endpos, lgt):
     if errors == 'replace':
         return '\xef\xbf\xbd', endpos, lgt + 1 # u'\ufffd'
@@ -105,30 +134,28 @@
         return chr3 == 0xa8 or chr3 == 0xa9
     return False
 
-# if you can't use the @elidable version, call str_check_utf8_impl()
-# directly
+class Utf8CheckError(Exception):
+    def __init__(self, msg, startpos, endpos):
+        self.msg = msg
+        self.startpos = startpos
+        self.endpos = endpos
+
 @jit.elidable
-def str_check_utf8(s, size, errors, final=False,
-                   errorhandler=None,
+def str_check_utf8(s, size, final=False,
                    allow_surrogates=runicode.allow_surrogate_by_default):
-    if errorhandler is None:
-        errorhandler = default_unicode_error_check
-    # XXX unclear, fix
+    """ A simplified version of utf8 encoder - it only works with 'strict'
+    error handling.
+    """
+    # XXX do the following in a cleaner way, e.g. via signature
     # NB. a bit messy because rtyper/rstr.py also calls the same
     # function.  Make sure we annotate for the args it passes, too
-    if NonConstant(False):
-        s = NonConstant('?????')
-        size = NonConstant(12345)
-        errors = NonConstant('strict')
-        final = NonConstant(True)
-        WTF # goes here
-        errorhandler = ll_unicode_error_decode
-        allow_surrogates = NonConstant(True)
-    return str_check_utf8_elidable(s, size, errors, final, errorhandler,
-                                   allow_surrogates=allow_surrogates)
-
-def str_check_utf8_impl(s, size, errors, final, errorhandler,
-                        allow_surrogates):
+    #if NonConstant(False):
+    #    s = NonConstant('?????')
+    #    size = NonConstant(12345)
+    #    errors = NonConstant('strict')
+    #    final = NonConstant(True)
+    #    errorhandler = ll_unicode_error_decode
+    #    allow_surrogates = NonConstant(True)
     if size == 0:
         return 0, 0
 
@@ -155,46 +182,43 @@
             # in case we need to continue running this loop
             if not charsleft:
                 # there's only the start byte and nothing else
-                errorhandler(errors, 'utf8', 'unexpected end of data',
-                             s, pos, pos+1)
+                raise Utf8CheckError('unexpected end of data', pos, pos + 1)
             ordch2 = ord(s[pos+1])
             if n == 3:
                 # 3-bytes seq with only a continuation byte
                 if runicode._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                     # second byte invalid, take the first and continue
-                    errorhandler(errors, 'utf8', 'invalid continuation byte',
-                                 s, pos, pos+1)
+                    raise Utf8CheckError('invalid continuation byte', pos,
+                                         pos + 1)
                 else:
                     # second byte valid, but third byte missing
-                    errorhandler(errors, 'utf8', 'unexpected end of data',
-                                 s, pos, pos+2)
+                    raise Utf8CheckError('unexpected end of data', pos, pos + 2)
             elif n == 4:
                 # 4-bytes seq with 1 or 2 continuation bytes
                 if runicode._invalid_byte_2_of_4(ordch1, ordch2):
                     # second byte invalid, take the first and continue
-                    errorhandler(errors, 'utf8', 'invalid continuation byte',
-                                 s, pos, pos+1)
+                    raise Utf8CheckError('invalid continuation byte', pos,
+                                         pos + 1)
                 elif charsleft == 2 and runicode._invalid_byte_3_of_4(ord(s[pos+2])):
                     # third byte invalid, take the first two and continue
-                    errorhandler(errors, 'utf8', 'invalid continuation byte',
-                                 s, pos, pos+2)
+                    raise Utf8CheckError('invalid continuation byte', pos,
+                                         pos + 2)
                 else:
                     # there's only 1 or 2 valid cb, but the others are missing
-                    errorhandler(errors, 'utf8', 'unexpected end of data',
-                                 s, pos, pos+charsleft+1)
+                    raise Utf8CheckError('unexpected end of data', pos,
+                                         pos + charsleft + 1)
             raise AssertionError("unreachable")
 
         if n == 0:
-            errorhandler(errors, 'utf8', 'invalid start byte', s, pos, pos+1)
+            raise Utf8CheckError('invalid start byte', pos, pos + 1)
         elif n == 1:
             assert 0, "ascii should have gone through the fast path"
 
         elif n == 2:
             ordch2 = ord(s[pos+1])
             if runicode._invalid_byte_2_of_2(ordch2):
-                errorhandler(errors, 'utf8', 'invalid continuation byte',
-                             s, pos, pos+2)
-                assert False, "unreachable"
+                raise Utf8CheckError('invalid continuation byte', pos,
+                                     pos + 2)
             # 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz
             lgt += 1
             pos += 2
@@ -203,13 +227,11 @@
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
             if runicode._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
-                errorhandler(errors, 'utf8', 'invalid continuation byte',
-                             s, pos, pos+1)
-                assert False, "unreachable"
+                raise Utf8CheckError('invalid continuation byte', pos,
+                                     pos + 1)
             elif runicode._invalid_byte_3_of_3(ordch3):
-                errorhandler(errors, 'utf8', 'invalid continuation byte',
-                             s, pos, pos+2)
-                assert False, "unreachable"
+                raise Utf8CheckError('invalid continuation byte', pos,
+                                     pos + 2)
             # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
             lgt += 1
             pos += 3
@@ -219,17 +241,14 @@
             ordch3 = ord(s[pos+2])
             ordch4 = ord(s[pos+3])
             if runicode._invalid_byte_2_of_4(ordch1, ordch2):
-                errorhandler(errors, 'utf8', 'invalid continuation byte',
-                             s, pos, pos+1)
-                assert False, "unreachable"
+                raise Utf8CheckError('invalid continuation byte', pos,
+                                     pos + 1)
             elif runicode._invalid_byte_3_of_4(ordch3):
-                errorhandler(errors, 'utf8', 'invalid continuation byte',
-                             s, pos, pos+2)
-                assert False, "unreachable"
+                raise Utf8CheckError('invalid continuation byte', pos,
+                                     pos + 2)
             elif runicode._invalid_byte_4_of_4(ordch4):
-                errorhandler(errors, 'utf8', 'invalid continuation byte',
-                             s, pos, pos+3)
-                assert False, "unreachable"
+                raise Utf8CheckError('invalid continuation byte', pos,
+                                     pos + 3)
             # 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz
             c = (((ordch1 & 0x07) << 18) +      # 0b00000111
                  ((ordch2 & 0x3F) << 12) +      # 0b00111111
@@ -243,5 +262,3 @@
             pos += 4
 
     return pos, lgt
-str_check_utf8_elidable = jit.elidable(
-    func_with_new_name(str_check_utf8_impl, "str_check_utf8_elidable"))

From pypy.commits at gmail.com  Sun Feb 26 17:42:15 2017
From: pypy.commits at gmail.com (fijal)
Date: Sun, 26 Feb 2017 14:42:15 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: sort out wrappers
Message-ID: <58b359c7.6718190a.d623e.f90c@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90370:b0612e5cd9da
Date: 2017-02-26 20:45 +0100
http://bitbucket.org/pypy/pypy/changeset/b0612e5cd9da/

Log:	sort out wrappers

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -109,14 +109,11 @@
         s, rs, p, lgt = self.orig(errors, encoding, msg, s.encode("utf8"), pos, endpos)
         return s, rs, p
 
-def utf8_encode_utf_7(utf8, utf8len, errors, errorhandler):
-    u = utf8.decode("utf8")
-    w = EncodeWrapper(errorhandler)
-    return runicode.unicode_encode_utf_7(u, len(u), errors, w.handle)
-
-def str_decode_utf_7(string, lgt, errors, final, errorhandler):
+# some irregular interfaces
+def str_decode_utf8(s, slen, errors, final, errorhandler):
     w = DecodeWrapper(errorhandler)
-    u, pos = runicode.str_decode_utf_7(string, lgt, errors, final, w.handle)
+    u, pos = runicode.str_decode_utf_8_impl(s, slen, errors, final, w.handle,
+        runicode.allow_surrogate_by_default)
     return u.encode('utf8'), pos, len(u)
 
 def str_decode_unicode_escape(s, slen, errors, final, errorhandler, ud_handler):
@@ -125,24 +122,30 @@
                                                 ud_handler)
     return u.encode('utf8'), pos, len(u)
 
-def str_decode_raw_unicode_escape(s, slen, errors, final, errorhandler):
-    w = DecodeWrapper(errorhandler)
-    u, pos = runicode.str_decode_raw_unicode_escape(s, slen, errors, final,
-                                                    w.handle)
-    return u.encode('utf8'), pos, len(u)
+def setup_new_encoders(encoding):
+    encoder_name = 'utf8_encode_' + encoding
+    encoder_call_name = 'unicode_encode_' + encoding
+    decoder_name = 'str_decode_' + encoding
+    def encoder(utf8, utf8len, errors, errorhandler):
+        u = utf8.decode("utf8")
+        w = EncodeWrapper(errorhandler)
+        return getattr(runicode, encoder_call_name)(u, len(u), errors,
+                       w.handle)
+    def decoder(s, slen, errors, final, errorhandler):
+        w = DecodeWrapper(errorhandler)
+        u, pos = getattr(runicode, decoder_name)(s, slen, errors, final, w.handle)
+        return u.encode('utf8'), pos, len(u)
+    encoder.__name__ = encoder_name
+    decoder.__name__ = decoder_name
+    if encoder_name not in globals():
+        globals()[encoder_name] = encoder
+    if decoder_name not in globals():
+        globals()[decoder_name] = decoder
 
-def str_decode_utf8(s, slen, errors, final, errorhandler):
-    w = DecodeWrapper(errorhandler)
-    u, pos = runicode.str_decode_utf_8_impl(s, slen, errors, final, w.handle,
-        runicode.allow_surrogate_by_default)
-    return u.encode('utf8'), pos, len(u)
+def setup():
+    for encoding in ['utf_7', 'unicode_escape', 'raw_unicode_escape',
+                     'utf_16', 'utf_16_le', 'utf_16_be', 'utf_32_le', 'utf_32',
+                     'utf_32_be', 'latin_1', 'unicode_internal']:
+        setup_new_encoders(encoding)
 
-def utf8_encode_utf_16(utf8, utf8len, errors, errorhandler):
-    w = EncodeWrapper(errorhandler)
-    u = utf8.decode("utf8")
-    return runicode.unicode_encode_utf_16(u, len(u), errors, w.handle)
-
-def utf8_encode_latin_1(utf8, utf8len, errors, errorhandler):
-    w = EncodeWrapper(errorhandler)
-    u = utf8.decode("utf8")
-    return runicode.unicode_encode_latin_1(u, len(u), errors, w.handle)
+setup()

From pypy.commits at gmail.com  Sun Feb 26 17:42:22 2017
From: pypy.commits at gmail.com (fijal)
Date: Sun, 26 Feb 2017 14:42:22 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: fixes
Message-ID: <58b359ce.a215190a.5fcbc.2484@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90373:77af71423e68
Date: 2017-02-26 23:41 +0100
http://bitbucket.org/pypy/pypy/changeset/77af71423e68/

Log:	fixes

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -604,6 +604,8 @@
 
 @unwrap_spec(string='bufferstr', errors='str_or_None')
 def charmap_decode(space, string, errors="strict", w_mapping=None):
+    from pypy.interpreter.unicodehelper import DecodeWrapper
+
     if errors is None:
         errors = 'strict'
     if len(string) == 0:
@@ -618,12 +620,13 @@
     state = space.fromcache(CodecState)
     result, consumed = runicode.str_decode_charmap(
         string, len(string), errors,
-        final, state.decode_error_handler, mapping)
+        final, DecodeWrapper(state.decode_error_handler).handle, mapping)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 @unwrap_spec(utf8='utf8', errors='str_or_None')
 def charmap_encode(space, utf8, utf8len, errors="strict", w_mapping=None):
-    xxx
+    from pypy.interpreter.unicodehelper import EncodeWrapper
+
     if errors is None:
         errors = 'strict'
     if space.is_none(w_mapping):
@@ -632,20 +635,24 @@
         mapping = Charmap_Encode(space, w_mapping)
 
     state = space.fromcache(CodecState)
+    uni = utf8.decode('utf8')
     result = runicode.unicode_encode_charmap(
         uni, len(uni), errors,
-        state.encode_error_handler, mapping)
+        EncodeWrapper(state.encode_error_handler).handle, mapping)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
 
 @unwrap_spec(chars='utf8')
 def charmap_build(space, chars, charslen):
     # XXX CPython sometimes uses a three-level trie
-    xxx
     w_charmap = space.newdict()
-    for num in range(len(chars)):
-        elem = chars[num]
-        space.setitem(w_charmap, space.newint(ord(elem)), space.newint(num))
+    pos = 0
+    num = 0
+    while num < charslen:
+        w_char = space.newint(rutf8.codepoint_at_pos(chars, pos))
+        space.setitem(w_charmap, w_char, space.newint(num))
+        pos = rutf8.next_codepoint_pos(chars, pos)
+        num += 1
     return w_charmap
 
 # ____________________________________________________________
@@ -690,6 +697,8 @@
 
 @unwrap_spec(errors='str_or_None')
 def unicode_internal_decode(space, w_string, errors="strict"):
+    from pypy.interpreter.unicodehelper import DecodeWrapper
+
     if errors is None:
         errors = 'strict'
     # special case for this codec: unicodes are returned as is
@@ -705,7 +714,7 @@
     state = space.fromcache(CodecState)
     result, consumed = runicode.str_decode_unicode_internal(
         string, len(string), errors,
-        final, state.decode_error_handler)
+        final, DecodeWrapper(state.decode_error_handler).handle)
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # ____________________________________________________________
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1166,8 +1166,8 @@
         c = mapping.get(ch, ERROR_CHAR)
         if c == ERROR_CHAR:
             r, pos = errorhandler(errors, "charmap",
-                                  "character maps to <undefined>",
-                                  s,  pos, pos + 1)
+                                       "character maps to <undefined>",
+                                       s,  pos, pos + 1)
             result.append(r)
             continue
         result.append(c)

From pypy.commits at gmail.com  Sun Feb 26 17:42:18 2017
From: pypy.commits at gmail.com (fijal)
Date: Sun, 26 Feb 2017 14:42:18 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: remove dead code and fix obscure
 length tracking bug
Message-ID: <58b359ca.5591190a.ec69.17bd@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90371:1c4b1dc9e28e
Date: 2017-02-26 21:35 +0100
http://bitbucket.org/pypy/pypy/changeset/1c4b1dc9e28e/

Log:	remove dead code and fix obscure length tracking bug

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1682,6 +1682,9 @@
     def unicode_w(self, w_obj):
         return self.utf8_w(w_obj).decode('utf8')
 
+    def realunicode_w(self, w_obj):
+        return self.realutf8_w(w_obj).decode('utf8')
+
     def newunicode(self, u):
         assert isinstance(u, unicode)
         return self.newutf8(u.encode("utf8"), len(u))
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -108,11 +108,11 @@
         return _create_list_from_unicode(self._value)
 
     def ord(self, space):
-        if len(self._value) != 1:
+        if self._len() != 1:
             raise oefmt(space.w_TypeError,
                          "ord() expected a character, but string of length %d "
                          "found", len(self._value))
-        return space.newint(ord(self._value[0]))
+        return space.newint(rutf8.codepoint_at_pos(self._utf8, 0))
 
     def _new(self, value):
         return W_UnicodeObject(value.encode('utf8'), len(value))
@@ -503,7 +503,7 @@
                     lgt += 1
             if keepends:
                 eol = pos
-                lgt += 2
+                lgt += 1
             strs_w.append(W_UnicodeObject(value[sol:eol], lgt))
         return space.newlist(strs_w)
 
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -66,6 +66,39 @@
         return pos + 1
     return pos + ord(runicode._utf8_code_length[chr1 - 0x80])
 
+def codepoint_at_pos(code, pos):
+    """ Give a codepoint in code at pos - assumes valid utf8, no checking!
+    """
+    import pdb
+    pdb.set_trace()
+    ordch1 = ord(code[pos])
+    if ordch1 < 0x80:
+        return ordch1
+
+    n = ord(runicode._utf8_code_length[ordch1 - 0x80])
+    if n == 2:
+        ordch2 = ord(code[pos+1])
+        # 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz
+        return (((ordch1 & 0x1F) << 6) +    # 0b00011111
+                 (ordch2 & 0x3F))           # 0b00111111
+    elif n == 3:
+        ordch2 = ord(code[pos+1])
+        ordch3 = ord(code[pos+2])
+        # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz
+        return (((ordch1 & 0x0F) << 12) +     # 0b00001111
+                ((ordch2 & 0x3F) << 6) +      # 0b00111111
+                (ordch3 & 0x3F))              # 0b00111111
+    elif n == 4:
+        ordch2 = ord(code[pos+1])
+        ordch3 = ord(code[pos+2])
+        ordch4 = ord(code[pos+3])
+        # 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz
+        return (((ordch1 & 0x07) << 18) +      # 0b00000111
+                ((ordch2 & 0x3F) << 12) +      # 0b00111111
+                ((ordch3 & 0x3F) << 6) +       # 0b00111111
+                (ordch4 & 0x3F))               # 0b00111111
+    assert False, "unreachable"
+
 class AsciiCheckError(Exception):
     def __init__(self, pos):
         self.pos = pos
@@ -75,9 +108,6 @@
         if ord(s[i]) & 0x80:
             raise AsciiCheckError(i)
 
-def default_unicode_error_check(*args):
-    xxx
-
 def utf8_encode_ascii(s, errors, encoding, msg, errorhandler):
     res = StringBuilder(len(s))
     u_pos = 0
@@ -109,14 +139,6 @@
         pos += 1
     return result.build(), pos, -1
 
-
-def default_unicode_error_decode(errors, encoding, message, s, pos, endpos, lgt):
-    if errors == 'replace':
-        return '\xef\xbf\xbd', endpos, lgt + 1 # u'\ufffd'
-    if errors == 'ignore':
-        return '', endpos, lgt
-    raise UnicodeDecodeError(encoding, s, pos, endpos, message)
-
 def check_newline_utf8(s, pos):
     chr1 = ord(s[pos])
     if 0xa <= chr1 <= 0xd:

From pypy.commits at gmail.com  Sun Feb 26 17:42:20 2017
From: pypy.commits at gmail.com (fijal)
Date: Sun, 26 Feb 2017 14:42:20 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: more consistency
Message-ID: <58b359cc.4395190a.2afe7.1105@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90372:cab3a879c6cb
Date: 2017-02-26 22:44 +0100
http://bitbucket.org/pypy/pypy/changeset/cab3a879c6cb/

Log:	more consistency

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -86,10 +86,10 @@
 
 def str_decode_ascii(s, slen, errors, final, errorhandler):
     try:
-        rutf8.check_ascii(s)
-        return s
+        rutf8.check_ascii(s, slen)
+        return s, slen, len(s)
     except rutf8.AsciiCheckError:
-        return rutf8.str_decode_ascii(s, errors, errorhandler)
+        return rutf8.str_decode_ascii(s, slen, errors, errorhandler)
 
 # XXX wrappers, think about speed
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -5,9 +5,7 @@
     enforceargs, newlist_hint)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder, split, rsplit, UnicodeBuilder
-from rpython.rlib.runicode import (
-    make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
-    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
+from rpython.rlib.runicode import make_unicode_escape_function
 from rpython.rlib import rutf8, jit
 
 from pypy.interpreter import unicodehelper
@@ -24,7 +22,7 @@
 
 __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode',
            'encode_object', 'decode_object', 'unicode_from_object',
-           'utf8_from_string', 'unicode_to_decimal_w']
+           'unicode_from_string', 'unicode_to_decimal_w']
 
 
 class W_UnicodeObject(W_Root):
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -69,8 +69,6 @@
 def codepoint_at_pos(code, pos):
     """ Give a codepoint in code at pos - assumes valid utf8, no checking!
     """
-    import pdb
-    pdb.set_trace()
     ordch1 = ord(code[pos])
     if ordch1 < 0x80:
         return ordch1
@@ -103,8 +101,10 @@
     def __init__(self, pos):
         self.pos = pos
 
-def check_ascii(s):
-    for i in range(0, len(s)):
+def check_ascii(s, size=-1):
+    if size == -1:
+        size = len(s)
+    for i in range(0, size):
         if ord(s[i]) & 0x80:
             raise AsciiCheckError(i)
 
@@ -123,9 +123,8 @@
         pos = next_codepoint_pos(s, pos)
     return res.build()
 
-def str_decode_ascii(s, errors, errorhandler):
+def str_decode_ascii(s, size, errors, errorhandler):
     # ASCII is equivalent to the first 128 ordinals in Unicode.
-    size = len(s)
     result = StringBuilder(size)
     pos = 0
     while pos < size:

From pypy.commits at gmail.com  Sun Feb 26 17:57:16 2017
From: pypy.commits at gmail.com (robert-zaremba)
Date: Sun, 26 Feb 2017 14:57:16 -0800 (PST)
Subject: [pypy-commit] pypy default: Build docs: fix fedora dependencies
 installation snippet.
Message-ID: <58b35d4c.4c152e0a.abc49.b8fe@mx.google.com>

Author: Robert Zaremba <robert.zaremba at scale-it.pl>
Branch: 
Changeset: r90374:3fbdd036db1c
Date: 2017-02-26 23:05 +0100
http://bitbucket.org/pypy/pypy/changeset/3fbdd036db1c/

Log:	Build docs: fix fedora dependencies installation snippet.

	In fedora the sqlite dependency should be sqlite-devel (lib-
	sqlite3-devel doesn't exists).

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -49,7 +49,7 @@
 -------------------------------
 (**Note**: for some hints on how to translate the Python interpreter under
 Windows, see the `windows document`_ . For hints on how to cross-compile in
-a chroot using scratchbox2, see the `arm document`_ in the 
+a chroot using scratchbox2, see the `arm document`_ in the
 `RPython documentation`_)
 
 .. _`windows document`: windows.html
@@ -57,7 +57,7 @@
 .. _`RPython documentation`: http://rpython.readthedocs.org
 
 The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
-already installed. If translating on CPython, you need to install it, e.g. 
+already installed. If translating on CPython, you need to install it, e.g.
 using ``pip install cffi``.
 
 To build PyPy on Unix using the C translation backend, you need at least a C
@@ -113,7 +113,7 @@
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
-    lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
+    sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
     gdbm-devel \
     xz-devel  # For lzma on PyPy3.
 
@@ -183,7 +183,7 @@
 imported the first time.
 
 ::
-    
+
     cd pypy/tool/release
     ./package.py pypy-VER-PLATFORM
 
@@ -220,5 +220,3 @@
 to continue normally. If the default path is usable, most code will be fine.
 However, the ``sys.prefix`` will be unset and some existing libraries assume
 that this is never the case.
-
-

From pypy.commits at gmail.com  Sun Feb 26 20:18:20 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Sun, 26 Feb 2017 17:18:20 -0800 (PST)
Subject: [pypy-commit] cffi sirtom67/float_complex: in test_c.py,
 add a test_call_function24 which
Message-ID: <58b37e5c.51d2190a.bc751.156a@mx.google.com>

Author: Tom Krauss <thomas.p.krauss at gmail.com>
Branch: sirtom67/float_complex
Changeset: r2893:7b2c8b6072c2
Date: 2017-02-26 19:17 -0600
http://bitbucket.org/cffi/cffi/changeset/7b2c8b6072c2/

Log:	in test_c.py, add a test_call_function24 which calls a C function
	that returns a "float _Complex"

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -62,6 +62,8 @@
 
 #include "malloc_closure.h"
 
+#include <complex.h>
+
 #if PY_MAJOR_VERSION >= 3
 # define STR_OR_BYTES "bytes"
 # define PyText_Type PyUnicode_Type
@@ -6588,6 +6590,10 @@
         return 1000 * p[0];
     return -42;
 }
+static float _Complex _testfunc24(float a, float b)
+{
+    return a + I*2.0*b;
+}
 
 static PyObject *b__testfunc(PyObject *self, PyObject *args)
 {
@@ -6621,6 +6627,7 @@
     case 21: f = &_testfunc21; break;
     case 22: f = &_testfunc22; break;
     case 23: f = &_testfunc23; break;
+    case 24: f = &_testfunc24; break;
     default:
         PyErr_SetNone(PyExc_ValueError);
         return NULL;
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1084,6 +1084,16 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
+def test_call_function_24():
+    BFloat = new_primitive_type("float")
+    BFloatComplex = new_primitive_type("float _Complex")
+    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
+    f = cast(BFunc3, _testfunc(24))
+    result = f(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact  
+
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")

From pypy.commits at gmail.com  Mon Feb 27 02:46:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 26 Feb 2017 23:46:23 -0800 (PST)
Subject: [pypy-commit] cffi default: Issue #310: pycparser was recently made
 stricter and no longer parses
Message-ID: <58b3d94f.12ce190a.e58af.258e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2894:85738d162c4f
Date: 2017-02-27 08:46 +0100
http://bitbucket.org/cffi/cffi/changeset/85738d162c4f/

Log:	Issue #310: pycparser was recently made stricter and no longer
	parses ``typedef int __dotdotdot__ foo_t;``. I think this fixes it

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -34,6 +34,9 @@
                               r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.')
 _r_star_const_space = re.compile(       # matches "* const "
     r"[*]\s*((const|volatile|restrict)\b\s*)+")
+_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+"
+                              r"\.\.\.")
+_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.")
 
 def _get_parser():
     global _parser_cache
@@ -180,6 +183,10 @@
             assert csource[p:p+3] == '...'
             csource = '%s __dotdotdot%d__ %s' % (csource[:p], number,
                                                  csource[p+3:])
+    # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__"
+    csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource)
+    # Replace "float ..." or "double..." with "__dotdotdotfloat__"
+    csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource)
     # Replace all remaining "..." with the same name, "__dotdotdot__",
     # which is declared with a typedef for the purpose of C parsing.
     return csource.replace('...', ' __dotdotdot__ '), macros
@@ -252,7 +259,8 @@
         typenames += sorted(ctn)
         #
         csourcelines = ['typedef int %s;' % typename for typename in typenames]
-        csourcelines.append('typedef int __dotdotdot__;')
+        csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
+                            ' __dotdotdot__;')
         csourcelines.append(csource)
         csource = '\n'.join(csourcelines)
         if lock is not None:
@@ -311,6 +319,8 @@
         for decl in iterator:
             if decl.name == '__dotdotdot__':
                 break
+        else:
+            assert 0
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
@@ -322,15 +332,15 @@
                         raise CDefError("typedef does not declare any name",
                                         decl)
                     quals = 0
-                    if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
-                            and decl.type.type.names[-1] == '__dotdotdot__'):
+                    if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and
+                            decl.type.type.names[-1].startswith('__dotdotdot')):
                         realtype = self._get_unknown_type(decl)
                     elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and
                           isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and
                           isinstance(decl.type.type.type,
                                      pycparser.c_ast.IdentifierType) and
-                          decl.type.type.type.names == ['__dotdotdot__']):
-                        realtype = model.unknown_ptr_type(decl.name)
+                          decl.type.type.type.names[-1].startswith('__dotdotdot')):
+                        realtype = self._get_unknown_ptr_type(decl)
                     else:
                         realtype, quals = self._get_type_and_quals(
                             decl.type, name=decl.name, partial_length_ok=True)
@@ -832,24 +842,25 @@
 
     def _get_unknown_type(self, decl):
         typenames = decl.type.type.names
-        assert typenames[-1] == '__dotdotdot__'
-        if len(typenames) == 1:
+        if typenames == ['__dotdotdot__']:
             return model.unknown_type(decl.name)
 
-        if (typenames[:-1] == ['float'] or
-            typenames[:-1] == ['double']):
-            # not for 'long double' so far
-            result = model.UnknownFloatType(decl.name)
-        else:
-            for t in typenames[:-1]:
-                if t not in ['int', 'short', 'long', 'signed',
-                             'unsigned', 'char']:
-                    raise FFIError(':%d: bad usage of "..."' %
-                                   decl.coord.line)
-            result = model.UnknownIntegerType(decl.name)
+        if typenames == ['__dotdotdotint__']:
+            if self._uses_new_feature is None:
+                self._uses_new_feature = "'typedef int... %s'" % decl.name
+            return model.UnknownIntegerType(decl.name)
 
-        if self._uses_new_feature is None:
-            self._uses_new_feature = "'typedef %s... %s'" % (
-                ' '.join(typenames[:-1]), decl.name)
+        if typenames == ['__dotdotdotfloat__']:
+            # note: not for 'long double' so far
+            if self._uses_new_feature is None:
+                self._uses_new_feature = "'typedef float... %s'" % decl.name
+            return model.UnknownFloatType(decl.name)
 
-        return result
+        raise FFIError(':%d: unsupported usage of "..." in typedef'
+                       % decl.coord.line)
+
+    def _get_unknown_ptr_type(self, decl):
+        if decl.type.type.type.names == ['__dotdotdot__']:
+            return model.unknown_ptr_type(decl.name)
+        raise FFIError(':%d: unsupported usage of "..." in typedef'
+                       % decl.coord.line)
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2246,10 +2246,10 @@
     assert str(e.value) == ("feature not supported with ffi.verify(), but only "
                             "with ffi.set_source(): 'typedef int... t1'")
     ffi = FFI()
-    ffi.cdef("typedef unsigned long... t1;")
+    ffi.cdef("typedef double ... t1;")
     e = py.test.raises(VerificationError, ffi.verify, "")
     assert str(e.value) == ("feature not supported with ffi.verify(), but only "
-                         "with ffi.set_source(): 'typedef unsigned long... t1'")
+                         "with ffi.set_source(): 'typedef float... t1'")
 
 def test_const_fields():
     ffi = FFI()

From pypy.commits at gmail.com  Mon Feb 27 04:27:45 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 27 Feb 2017 01:27:45 -0800 (PST)
Subject: [pypy-commit] pypy default: Ignore cffi build leftovers.
Message-ID: <58b3f111.42452e0a.5a92c.e12e@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: 
Changeset: r90375:44d9fc927ace
Date: 2017-02-27 10:28 +0100
http://bitbucket.org/pypy/pypy/changeset/44d9fc927ace/

Log:	Ignore cffi build leftovers.

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -60,6 +60,9 @@
 ^lib_pypy/ctypes_config_cache/_.+_cache\.py$
 ^lib_pypy/ctypes_config_cache/_.+_.+_\.py$
 ^lib_pypy/_libmpdec/.+.o$
+^lib_pypy/.+.c$
+^lib_pypy/.+.o$
+^lib_pypy/.+.so$
 ^pypy/doc/discussion/.+\.html$
 ^include/.+\.h$
 ^include/.+\.inl$

From pypy.commits at gmail.com  Mon Feb 27 06:11:59 2017
From: pypy.commits at gmail.com (tobweber)
Date: Mon, 27 Feb 2017 03:11:59 -0800 (PST)
Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Created new
 branch c8-efficient-serial-execution
Message-ID: <58b4097f.53532e0a.7a9e3.ccb0@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-efficient-serial-execution
Changeset: r2015:5f37db83b9d8
Date: 2017-02-27 11:11 +0000
http://bitbucket.org/pypy/stmgc/changeset/5f37db83b9d8/

Log:	Created new branch c8-efficient-serial-execution


From pypy.commits at gmail.com  Mon Feb 27 06:12:22 2017
From: pypy.commits at gmail.com (fijal)
Date: Mon, 27 Feb 2017 03:12:22 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: (fijal,
 argio) whack until we run into a serious problem
Message-ID: <58b40996.5591190a.ec69.3183@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90376:85fee86ba1f7
Date: 2017-02-27 12:11 +0100
http://bitbucket.org/pypy/pypy/changeset/85fee86ba1f7/

Log:	(fijal, argio) whack until we run into a serious problem

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -10,7 +10,7 @@
 
     interpleveldefs = {
         "StringBuilder": "interp_builders.W_StringBuilder",
-        "UnicodeBuilder": "interp_builders.W_UnicodeBuilder",
+        #"UnicodeBuilder": "interp_builders.W_UnicodeBuilder",
     }
 
 class TimeModule(MixedModule):
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -64,4 +64,4 @@
     return W_Builder
 
 W_StringBuilder = create_builder("StringBuilder", str, StringBuilder, "newbytes")
-W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder, "newunicode")
+#W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder, "newunicode")
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -486,6 +486,7 @@
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
+    assert False, "fix in the future"
     if errors is None:
         errors = 'strict'
     final = space.is_true(w_final)
@@ -507,6 +508,7 @@
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
 def utf_32_ex_decode(space, data, errors='strict', byteorder=0, w_final=None):
+    assert False, "fix in the future"
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
     if byteorder == 0:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -47,8 +47,8 @@
         return NonConstant("foobar")
     identifier_w = bytes_w = str_w
 
-    def unicode_w(self, space):
-        return NonConstant(u"foobar")
+    def utf8_w(self, space):
+        return NonConstant("foobar")
 
     def int_w(self, space, allow_conversion=True):
         return NonConstant(-42)
@@ -208,7 +208,7 @@
     def newbytes(self, x):
         return w_some_obj()
 
-    def newunicode(self, x):
+    def newutf8(self, x, l):
         return w_some_obj()
 
     newtext = newbytes
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -17,7 +17,7 @@
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.unicodeobject import (
     decode_object, unicode_from_encoded_object,
-    getdefaultencoding)
+    getdefaultencoding, unicode_from_string)
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 
 
@@ -53,17 +53,7 @@
         return space.newint(uid)
 
     def convert_to_w_unicode(self, space):
-        # Use the default encoding.
-        encoding = getdefaultencoding(space)
-        if encoding == 'ascii':
-            try:
-                rutf8.check_ascii(self._value)
-                return space.newutf8(self._value, len(self._value))
-            except rutf8.AsciiCheckError:
-                xxx
-        else:
-            xxx
-        return space.unicode_w(decode_object(space, self, encoding, None))
+        return unicode_from_string(space, self)
 
     def descr_add(self, space, w_other):
         """x.__add__(y) <==> x+y"""
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1032,7 +1032,7 @@
 
         unilist = space.listview_unicode(w_iterable)
         if unilist is not None:
-            xxx
+            assert False, "disabled"
             w_list.strategy = strategy = space.fromcache(UnicodeListStrategy)
             # need to copy because intlist can share with w_iterable
             w_list.lstorage = strategy.erase(unilist[:])
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -203,6 +203,7 @@
         return unichr(unicodedb.totitle(ord(ch)))
 
     def _newlist_unwrapped(self, space, lst):
+        assert False, "should not be called"
         return space.newlist_unicode(lst)
 
     @staticmethod
diff --git a/rpython/annotator/listdef.py b/rpython/annotator/listdef.py
--- a/rpython/annotator/listdef.py
+++ b/rpython/annotator/listdef.py
@@ -107,6 +107,9 @@
             self.bookkeeper.annotator.reflowfromposition(position_key)
 
     def generalize(self, s_other_value):
+        if hasattr(self.s_value, 'can_be_None') and not self.s_value.can_be_None and getattr(s_other_value, 'can_be_None', False):
+            import pdb
+            pdb.set_trace()
         s_new_value = unionof(self.s_value, s_other_value)
         updated = s_new_value != self.s_value
         if updated:
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -671,7 +671,7 @@
         return getbookkeeper().newlist(s_item)
 
     def method_rsplit(self, patt, max=-1):
-        s_item = self.basestringclass(no_nul=self.no_nul)
+        s_item = self.basestringclass(no_nul=self.no_nul, can_be_None=False)
         return getbookkeeper().newlist(s_item)
 
     def method_replace(self, s1, s2):
@@ -696,7 +696,7 @@
         if not s_enc.is_constant():
             raise AnnotatorError("Non-constant encoding not supported")
         enc = s_enc.const
-        if enc not in ('ascii', 'latin-1', 'utf-8'):
+        if enc not in ('ascii', 'latin-1', 'utf-8', 'utf8'):
             raise AnnotatorError("Encoding %s not supported for unicode" % (enc,))
         return SomeString(no_nul=self.no_nul)
     method_encode.can_only_throw = []
@@ -729,7 +729,7 @@
         if not s_enc.is_constant():
             raise AnnotatorError("Non-constant encoding not supported")
         enc = s_enc.const
-        if enc not in ('ascii', 'latin-1', 'utf-8'):
+        if enc not in ('ascii', 'latin-1', 'utf-8', 'utf8'):
             raise AnnotatorError("Encoding %s not supported for strings" % (enc,))
         return SomeUnicodeString(no_nul=self.no_nul)
     method_decode.can_only_throw = [UnicodeDecodeError]
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -346,8 +346,6 @@
     #
     # See also unicode_encode_utf8sp().
     #
-    if errorhandler is None:
-        errorhandler = default_unicode_error_encode
     # NB. a bit messy because rtyper/rstr.py also calls the same
     # function.  Make sure we annotate for the args it passes, too
     if NonConstant(False):
@@ -361,6 +359,9 @@
 
 def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                               allow_surrogates=False):
+    # XXX hack
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
     assert(size >= 0)
     result = StringBuilder(size)
     pos = 0
diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -708,7 +708,9 @@
                     assert stop >= 0
                     chunks.append(self.buf[:stop])
                     break
-                chunks.append(self.buf)
+                buf = self.buf
+                assert buf is not None
+                chunks.append(buf)
             return ''.join(chunks)
 
     def readline(self):
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -335,7 +335,7 @@
             return hop.gendirectcall(self.ll.ll_str2unicode, v_self)
         elif encoding == 'latin-1':
             return hop.gendirectcall(self.ll_decode_latin1, v_self)
-        elif encoding == 'utf-8':
+        elif encoding == 'utf-8' or encoding == 'utf8':
             return hop.gendirectcall(self.ll_decode_utf8, v_self)
         else:
             raise TyperError("encoding %s not implemented" % (encoding, ))
@@ -408,7 +408,7 @@
             return hop.gendirectcall(self.ll_str, v_self)
         elif encoding == "latin-1":
             return hop.gendirectcall(self.ll_encode_latin1, v_self)
-        elif encoding == 'utf-8':
+        elif encoding == 'utf-8' or encoding == 'utf8':
             return hop.gendirectcall(self.ll_encode_utf8, v_self)
         else:
             raise TyperError("encoding %s not implemented" % (encoding, ))

From pypy.commits at gmail.com  Mon Feb 27 06:39:49 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:39:49 -0800 (PST)
Subject: [pypy-commit] pypy get/setpriority: getpriority and setpriority
 attributes
Message-ID: <58b41005.94472e0a.51268.9aa3@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: get/setpriority
Changeset: r90377:e245c0e11157
Date: 2016-12-27 23:45 +0300
http://bitbucket.org/pypy/pypy/changeset/e245c0e11157/

Log:	getpriority and setpriority attributes

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -169,7 +169,8 @@
                  'setpgrp', 'getppid', 'getpgid', 'setpgid', 'setreuid',
                  'setregid', 'getsid', 'setsid', 'fstatvfs', 'statvfs',
                  'setgroups', 'initgroups', 'tcgetpgrp', 'tcsetpgrp',
-                 'getresuid', 'getresgid', 'setresuid', 'setresgid']:
+                 'getresuid', 'getresgid', 'setresuid', 'setresgid',
+                 'getpriority', 'setpriority']:
         if hasattr(os, name):
             interpleveldefs[name] = 'interp_posix.%s' % (name,)
     # not visible via os, inconsistency in nt:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1605,6 +1605,29 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
+ at unwrap_spec(program=int, identifier=int)
+def getpriority(space, program, identifier):
+    """ getpriority(process, identifier) -> current_priority
+    
+    Get program scheduling priority.
+    """
+    try:
+        returned_priority = os.getpriority(process, identifier)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+    return space.wrap(returned_priority)
+
+ at unwrap_spec(program=int, identifier=int, priority=int)
+def setpriority(space, program, identifier, priority):
+    """ setpriority(process, identifier) 
+    
+    Set program scheduling priority.
+    """
+    try:
+        os.setpriority(process, identifier, priority)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+        
 @unwrap_spec(path='fsencode')
 def chroot(space, path):
     """ chroot(path)
@@ -2241,4 +2264,4 @@
             flags |= rposix.O_NONBLOCK
         rposix.set_status_flags(fd, flags)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e)
\ No newline at end of file
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -850,6 +850,17 @@
         assert st.st_size == 10000000000
     test_largefile.need_sparse_files = True
 
+    def test_os_getpriority(self):
+        posix, os = self.posix, self.os
+        assert os.getpriority(os.PRIO_PROCESS, os.getpid()) == posix.getpriority(os.PRIO_PROCESS, os.getpid())
+
+    def test_os_setpriority(self):
+        os = self.posix
+        result = os.getpriority(os.PRIO_PROCESS, os.getpid())
+        os.setpriority(os.PRIO_PROCESS, os.getpid(), result + 1)
+        new_result = os.getpriority(os.PRIO_PROCESS, os.getpid())
+        assert new_result == (result + 1)
+
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',

From pypy.commits at gmail.com  Mon Feb 27 06:39:51 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:39:51 -0800 (PST)
Subject: [pypy-commit] pypy get/setpriority: registering attributes
Message-ID: <58b41007.4c142e0a.d17e0.df1d@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: get/setpriority
Changeset: r90378:56478fb6d837
Date: 2016-12-28 02:15 +0300
http://bitbucket.org/pypy/pypy/changeset/56478fb6d837/

Log:	registering attributes

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -81,6 +81,8 @@
         'scandir': 'interp_scandir.scandir',
         'get_inheritable': 'interp_posix.get_inheritable',
         'set_inheritable': 'interp_posix.set_inheritable',
+        'getpriority': 'interp_posix.getpriority',
+        'setpriority': 'interp_posix.setpriority',
     }
 
     if hasattr(os, 'chown'):
@@ -169,8 +171,7 @@
                  'setpgrp', 'getppid', 'getpgid', 'setpgid', 'setreuid',
                  'setregid', 'getsid', 'setsid', 'fstatvfs', 'statvfs',
                  'setgroups', 'initgroups', 'tcgetpgrp', 'tcsetpgrp',
-                 'getresuid', 'getresgid', 'setresuid', 'setresgid',
-                 'getpriority', 'setpriority']:
+                 'getresuid', 'getresgid', 'setresuid', 'setresgid',]:
         if hasattr(os, name):
             interpleveldefs[name] = 'interp_posix.%s' % (name,)
     # not visible via os, inconsistency in nt:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -855,10 +855,10 @@
         assert os.getpriority(os.PRIO_PROCESS, os.getpid()) == posix.getpriority(os.PRIO_PROCESS, os.getpid())
 
     def test_os_setpriority(self):
-        os = self.posix
-        result = os.getpriority(os.PRIO_PROCESS, os.getpid())
-        os.setpriority(os.PRIO_PROCESS, os.getpid(), result + 1)
-        new_result = os.getpriority(os.PRIO_PROCESS, os.getpid())
+        posix, os = self.posix, self.os
+        result = posix.getpriority(os.PRIO_PROCESS, os.getpid())
+        posix.setpriority(os.PRIO_PROCESS, os.getpid(), result + 1)
+        new_result = posix.getpriority(os.PRIO_PROCESS, os.getpid())
         assert new_result == (result + 1)
 
     def test_write_buffer(self):

From pypy.commits at gmail.com  Mon Feb 27 06:39:58 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:39:58 -0800 (PST)
Subject: [pypy-commit] pypy get/setpriority: exposed prio_process, prio_user,
 prio_pgrp
Message-ID: <58b4100e.cf4d2e0a.f521.a4d6@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: get/setpriority
Changeset: r90381:fcaf312e010c
Date: 2017-01-09 14:06 +0300
http://bitbucket.org/pypy/pypy/changeset/fcaf312e010c/

Log:	exposed prio_process,prio_user,prio_pgrp

diff --git a/lib-python/2.7/os.py b/lib-python/2.7/os.py
--- a/lib-python/2.7/os.py
+++ b/lib-python/2.7/os.py
@@ -124,12 +124,11 @@
 
 # Python uses fixed values for the SEEK_ constants; they are mapped
 # to native constants if necessary in posixmodule.c
+# Other possible SEEK values are directly imported from posixmodule.c
 SEEK_SET = 0
 SEEK_CUR = 1
 SEEK_END = 2
 
-#'
-
 # Super directory utilities.
 # (Inspired by Eric Raymond; the doc strings are mostly his)
 
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -83,6 +83,9 @@
         'set_inheritable': 'interp_posix.set_inheritable',
         'getpriority': 'interp_posix.getpriority',
         'setpriority': 'interp_posix.setpriority',
+        'PRIO_PROCESS': 'space.newint(0)',
+        'PRIO_PGRP': 'space.newint(1)',
+        'PRIO_USER': 'space.newint(2)',
     }
 
     if hasattr(os, 'chown'):
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1847,7 +1847,7 @@
     Get program scheduling priority.
     """
     try:
-        returned_priority = rposix.getpriority(program, identifier)
+        returned_priority = rposix.c_getpriority(program, identifier)
     except OSError as e:
         raise wrap_oserror(space, e)
     return space.wrap(returned_priority)
@@ -1859,7 +1859,7 @@
     Set program scheduling priority.
     """
     try:
-        rposix.setpriority(program, identifier, priority)
+        rposix.c_setpriority(program, identifier, priority)
     except OSError as e:
         raise wrap_oserror(space, e)
 
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -850,16 +850,21 @@
         assert st.st_size == 10000000000
     test_largefile.need_sparse_files = True
 
-    def test_os_getpriority(self):
+    def test_os_set_get_priority(self):
         posix, os = self.posix, self.os
-        assert os.getpriority(0,os.getpid()) == posix.getpriority(0,os.getpid())
-
-    def test_os_setpriority(self):
-        posix, os = self.posix, self.os
-        orig_priority = posix.getpriority(0, os.getpid())
-        posix.setpriority(0, os.getpid(), orig_priority + 1)
-        new_result = posix.getpriority(0, os.getpid())
-        assert new_result == (orig_priority + 1)
+        orig_priority = posix.getpriority(posix.PRIO_PROCESS, os.getpid())
+        posix.setpriority(posix.PRIO_PROCESS, os.getpid(), orig_priority + 1)
+        assert orig_priority == 0
+        assert posix.getpriority(posix.PRIO_PGRP, os.getpid()) == 0
+        try:
+            new_result = posix.getpriority(posix.PRIO_PROCESS, os.getpid())
+            assert new_result == (orig_priority + 1)
+        finally:
+            try:
+                posix.setpriority(posix.PRIO_PROCESS, posix.getpid(), orig_priority)
+            except OSError as err:
+                if err.errno != errno.EACCES:
+                    raise
 
     def test_write_buffer(self):
         os = self.posix
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -249,6 +249,9 @@
     SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
     SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
     SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
+    PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
+    PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
+    PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
@@ -1738,15 +1741,7 @@
                           save_err=rffi.RFFI_SAVE_ERRNO)
     c_setpriority = external('setpriority', [rffi.INT, rffi.INT, rffi.INT], rffi.INT,
                           save_err=rffi.RFFI_SAVE_ERRNO)
-
-    @replace_os_function('getpriority')
-    def getpriority(program, identifier):
-        return handle_posix_error('getpriority', c_getpriority(program, identifier))
-
-    @replace_os_function('setpriority')
-    def setpriority(program, identifier, priority):
-        handle_posix_error('setpriority', c_setpriority(program, identifier, priority))
-
+    
 #___________________________________________________________________
 
 c_chroot = external('chroot', [rffi.CCHARP], rffi.INT,
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -663,3 +663,4 @@
     finally:
         os.close(fd1)
         os.close(fd2)
+        
\ No newline at end of file

From pypy.commits at gmail.com  Mon Feb 27 06:39:53 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:39:53 -0800 (PST)
Subject: [pypy-commit] pypy get/setpriority: translate to corresponding C
 code
Message-ID: <58b41009.5125190a.807a2.9faa@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: get/setpriority
Changeset: r90379:a36386624a08
Date: 2016-12-30 17:29 +0300
http://bitbucket.org/pypy/pypy/changeset/a36386624a08/

Log:	translate to corresponding C code

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1605,29 +1605,6 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(program=int, identifier=int)
-def getpriority(space, program, identifier):
-    """ getpriority(process, identifier) -> current_priority
-    
-    Get program scheduling priority.
-    """
-    try:
-        returned_priority = os.getpriority(process, identifier)
-    except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(returned_priority)
-
- at unwrap_spec(program=int, identifier=int, priority=int)
-def setpriority(space, program, identifier, priority):
-    """ setpriority(process, identifier) 
-    
-    Set program scheduling priority.
-    """
-    try:
-        os.setpriority(process, identifier, priority)
-    except OSError as e:
-        raise wrap_oserror(space, e)
-        
 @unwrap_spec(path='fsencode')
 def chroot(space, path):
     """ chroot(path)
@@ -1863,6 +1840,29 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
+ at unwrap_spec(program=int, identifier=int)
+def getpriority(space, program, identifier):
+    """ getpriority(process, identifier) -> int
+    
+    Get program scheduling priority.
+    """
+    try:
+        returned_priority = rposix.getpriority(program, identifier)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+    return space.wrap(returned_priority)
+
+ at unwrap_spec(program=int, identifier=int, priority=int)
+def setpriority(space, program, identifier, priority):
+    """ setpriority(process, identifier) 
+    
+    Set program scheduling priority.
+    """
+    try:
+        rposix.setpriority(program, identifier, priority)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+
 def declare_new_w_star(name):
     if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
         @unwrap_spec(status=c_int)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -852,15 +852,15 @@
 
     def test_os_getpriority(self):
         posix, os = self.posix, self.os
-        assert os.getpriority(os.PRIO_PROCESS, os.getpid()) == posix.getpriority(os.PRIO_PROCESS, os.getpid())
+        assert os.getpriority(1,os.getpid()) == posix.getpriority(1,os.getpid())
 
     def test_os_setpriority(self):
         posix, os = self.posix, self.os
-        result = posix.getpriority(os.PRIO_PROCESS, os.getpid())
-        posix.setpriority(os.PRIO_PROCESS, os.getpid(), result + 1)
-        new_result = posix.getpriority(os.PRIO_PROCESS, os.getpid())
-        assert new_result == (result + 1)
-
+        orig_priority = posix.getpriority(1, os.getpid())
+        posix.setpriority(1, os.getpid(), orig_priority + 1)
+        new_result = posix.getpriority(1, os.getpid())
+        assert new_result == (orig_priority + 1)
+        
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1734,6 +1734,19 @@
     def setresgid(rgid, egid, sgid):
         handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
+    c_getpriority = external('getpriority', [rffi.INT, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+    c_setpriority = external('setpriority', [rffi.INT, rffi.INT, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @replace_os_function('getpriority')
+    def getpriority(program, identifier):
+        return handle_posix_error('getpriority', c_getpriority(program, identifier))
+
+    @replace_os_function('setpriority')
+    def setpriority(program, identifier, priority):
+        handle_posix_error('setpriority', c_setpriority(program, identifier, priority))
+
 #___________________________________________________________________
 
 c_chroot = external('chroot', [rffi.CCHARP], rffi.INT,

From pypy.commits at gmail.com  Mon Feb 27 06:39:56 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:39:56 -0800 (PST)
Subject: [pypy-commit] pypy get/setpriority: change in test data
Message-ID: <58b4100c.0d57190a.e5f64.2b50@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: get/setpriority
Changeset: r90380:2d3af8f1bada
Date: 2016-12-30 17:46 +0300
http://bitbucket.org/pypy/pypy/changeset/2d3af8f1bada/

Log:	change in test data

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -852,15 +852,15 @@
 
     def test_os_getpriority(self):
         posix, os = self.posix, self.os
-        assert os.getpriority(1,os.getpid()) == posix.getpriority(1,os.getpid())
+        assert os.getpriority(0,os.getpid()) == posix.getpriority(0,os.getpid())
 
     def test_os_setpriority(self):
         posix, os = self.posix, self.os
-        orig_priority = posix.getpriority(1, os.getpid())
-        posix.setpriority(1, os.getpid(), orig_priority + 1)
-        new_result = posix.getpriority(1, os.getpid())
+        orig_priority = posix.getpriority(0, os.getpid())
+        posix.setpriority(0, os.getpid(), orig_priority + 1)
+        new_result = posix.getpriority(0, os.getpid())
         assert new_result == (orig_priority + 1)
-        
+
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',

From pypy.commits at gmail.com  Mon Feb 27 06:40:04 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:40:04 -0800 (PST)
Subject: [pypy-commit] pypy pread/pwrite: OFF_T
Message-ID: <58b41014.5038190a.7c0b5.3664@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: pread/pwrite
Changeset: r90384:074c9e98df7c
Date: 2017-02-20 12:05 +0300
http://bitbucket.org/pypy/pypy/changeset/074c9e98df7c/

Log:	OFF_T

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -11,7 +11,7 @@
     _prefer_unicode, _preferred_traits)
 from rpython.rlib.objectmodel import (
     specialize, enforceargs, register_replacement_for, NOT_CONSTANT)
-from rpython.rlib.rarithmetic import intmask, widen
+from rpython.rlib.rarithmetic import intmask, widen, r_longlong
 from rpython.rlib.signature import signature
 from rpython.tool.sourcetools import func_renamer
 from rpython.translator.platform import platform

From pypy.commits at gmail.com  Mon Feb 27 06:40:02 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:40:02 -0800 (PST)
Subject: [pypy-commit] pypy pread/pwrite: pread/pwrite
Message-ID: <58b41012.0163190a.fe27b.2f10@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: pread/pwrite
Changeset: r90383:9762c2f4cd64
Date: 2017-02-18 21:03 +0300
http://bitbucket.org/pypy/pypy/changeset/9762c2f4cd64/

Log:	pread/pwrite

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -206,6 +206,11 @@
         interpleveldefs['get_blocking'] = 'interp_posix.get_blocking'
         interpleveldefs['set_blocking'] = 'interp_posix.set_blocking'
 
+    if hasattr(rposix, 'pread'):
+        interpleveldefs['pread'] = 'interp_posix.pread'
+    if hasattr(rposix, 'pwrite'):
+        interpleveldefs['pwrite'] = 'interp_posix.pwrite'
+        
     for _name in ["O_CLOEXEC"]:
         if getattr(rposix, _name) is not None:
             interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -269,7 +269,7 @@
         raise wrap_oserror(space, e)
     else:
         return space.newbytes(s)
-
+        
 @unwrap_spec(fd=c_int)
 def write(space, fd, w_data):
     """Write a string to a file descriptor.  Return the number of bytes
@@ -282,6 +282,25 @@
     else:
         return space.wrap(res)
 
+ at unwrap_spec(fd=c_int, length=int, offset=int)
+def pread(space, fd, length, offset):
+    try:
+        s = rposix.pread(fd, length, offset)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+    else:
+        return space.newbytes(s)
+
+ at unwrap_spec(fd=c_int, offset=int)
+def pwrite(space, fd, w_data, offset):
+    data = space.getarg_w('y*', w_data)
+    try:
+        res = rposix.pwrite(fd, data.as_str(), offset)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+    else:
+        return space.wrap(res)
+
 @unwrap_spec(fd=c_int)
 def close(space, fd):
     """Close a file descriptor (for low level IO)."""
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -833,6 +833,30 @@
                     os.chdir(localdir)
             raises(ValueError, os.fchdir, -1)
 
+    if hasattr(rposix, 'pread'):
+        def test_os_pread(self):
+            os = self.posix
+            fd = os.open(self.path2 + 'test_os_pread', os.O_RDWR | os.O_CREAT)
+            try:
+                os.write(fd, b'test')
+                os.lseek(fd, 0, os.SEEK_SET)
+                assert os.pread(fd, 2, 1) == b'es'
+                assert os.read(fd, 2) == b'te'
+            finally:
+                os.close(fd)
+
+    if hasattr(rposix, 'pwrite'):
+        def test_os_pwrite(self):
+            os = self.posix
+            fd = os.open(self.path2 + 'test_os_pwrite', os.O_RDWR | os.O_CREAT)
+            try:
+                os.write(fd, b'test')
+                os.lseek(fd, 0, os.SEEK_SET)
+                os.pwrite(fd, b'xx', 1)
+                assert os.read(fd, 4) == b'txxt'
+            finally:
+                os.close(fd)
+                
     def test_largefile(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_largefile',
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -485,6 +485,31 @@
 def sync():
     c_sync()
 
+c_pread = external('pread',
+                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T , rffi.LONGLONG], rffi.SSIZE_T,
+                  save_err=rffi.RFFI_SAVE_ERRNO)
+c_pwrite = external('pwrite',
+                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T, rffi.LONGLONG], rffi.SSIZE_T,
+                   save_err=rffi.RFFI_SAVE_ERRNO)
+
+ at replace_os_function('pread')
+ at enforceargs(int, int, None)
+def pread(fd, count, offset):
+    if count < 0:
+        raise OSError(errno.EINVAL, None)
+    validate_fd(fd)
+    with rffi.scoped_alloc_buffer(count) as buf:
+        void_buf = rffi.cast(rffi.VOIDP, buf.raw)
+        return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset)))
+
+ at replace_os_function('pwrite')
+ at enforceargs(int, None, None)
+def pwrite(fd, data, offset):
+    count = len(data)
+    validate_fd(fd)
+    with rffi.scoped_nonmovingbuffer(data) as buf:
+        return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset))
+
 #___________________________________________________________________
 
 c_chdir = external('chdir', [rffi.CCHARP], rffi.INT,

From pypy.commits at gmail.com  Mon Feb 27 06:40:09 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:40:09 -0800 (PST)
Subject: [pypy-commit] pypy pread/pwrite: removed unwanted import
Message-ID: <58b41019.5125190a.807a2.9fae@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: pread/pwrite
Changeset: r90386:a9fa1c5caee1
Date: 2017-02-20 19:28 +0300
http://bitbucket.org/pypy/pypy/changeset/a9fa1c5caee1/

Log:	removed unwanted import

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -11,7 +11,7 @@
     _prefer_unicode, _preferred_traits)
 from rpython.rlib.objectmodel import (
     specialize, enforceargs, register_replacement_for, NOT_CONSTANT)
-from rpython.rlib.rarithmetic import intmask, widen, r_longlong
+from rpython.rlib.rarithmetic import intmask, widen
 from rpython.rlib.signature import signature
 from rpython.tool.sourcetools import func_renamer
 from rpython.translator.platform import platform

From pypy.commits at gmail.com  Mon Feb 27 06:40:06 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:40:06 -0800 (PST)
Subject: [pypy-commit] pypy pread/pwrite: using OFF_T instead of
 rffi.LONGLONG
Message-ID: <58b41016.1a0e2e0a.4eca.d522@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: pread/pwrite
Changeset: r90385:b920b22b16fd
Date: 2017-02-20 19:14 +0300
http://bitbucket.org/pypy/pypy/changeset/b920b22b16fd/

Log:	using OFF_T instead of rffi.LONGLONG

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -250,6 +250,7 @@
     SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
     SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
+    OFF_T = rffi_platform.SimpleType('off_t')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
@@ -486,10 +487,10 @@
     c_sync()
 
 c_pread = external('pread',
-                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T , rffi.LONGLONG], rffi.SSIZE_T,
+                  [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T,
                   save_err=rffi.RFFI_SAVE_ERRNO)
 c_pwrite = external('pwrite',
-                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T, rffi.LONGLONG], rffi.SSIZE_T,
+                   [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T,
                    save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('pread')

From pypy.commits at gmail.com  Mon Feb 27 06:40:11 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:40:11 -0800 (PST)
Subject: [pypy-commit] pypy pread/pwrite: retry transparently when we get
 EINTR error
Message-ID: <58b4101b.4d29190a.6d6ec.46a7@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: pread/pwrite
Changeset: r90387:c61e7a2fca32
Date: 2017-02-25 13:18 +0300
http://bitbucket.org/pypy/pypy/changeset/c61e7a2fca32/

Log:	retry transparently when we get EINTR error

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -206,11 +206,18 @@
         interpleveldefs['get_blocking'] = 'interp_posix.get_blocking'
         interpleveldefs['set_blocking'] = 'interp_posix.set_blocking'
 
+    if hasattr(rposix, 'getpriority'):
+        interpleveldefs['getpriority'] = 'interp_posix.getpriority'
+        interpleveldefs['setpriority'] = 'interp_posix.setpriority'
+        for _name in ['PRIO_PROCESS', 'PRIO_PGRP', 'PRIO_USER']:
+            assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
+            interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
+
     if hasattr(rposix, 'pread'):
         interpleveldefs['pread'] = 'interp_posix.pread'
     if hasattr(rposix, 'pwrite'):
-        interpleveldefs['pwrite'] = 'interp_posix.pwrite'
-        
+       interpleveldefs['pwrite'] = 'interp_posix.pwrite'
+       
     for _name in ["O_CLOEXEC"]:
         if getattr(rposix, _name) is not None:
             interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
@@ -225,4 +232,9 @@
 for constant in dir(os):
     value = getattr(os, constant)
     if constant.isupper() and type(value) is int:
+        if constant in ['SEEK_SET', 'SEEK_CUR', 'SEEK_END',
+                        'P_NOWAIT', 'P_NOWAITO', 'P_WAIT']:
+            # obscure, but these names are not in CPython's posix module
+            # and if we put it here then they end up twice in 'os.__all__'
+            continue
         Module.interpleveldefs[constant] = "space.wrap(%s)" % value
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -40,9 +40,9 @@
 
 def wrap_uid(space, uid):
     if uid <= r_uint(sys.maxint):
-        return space.wrap(intmask(uid))
+        return space.newint(intmask(uid))
     else:
-        return space.wrap(uid)     # an unsigned number
+        return space.newint(uid)     # an unsigned number
 wrap_gid = wrap_uid
 
 class FileEncoder(object):
@@ -66,7 +66,7 @@
         self.w_obj = w_obj
 
     def as_bytes(self):
-        return self.space.fsencode_w(self.w_obj)
+        return self.space.bytes0_w(self.w_obj)
 
     def as_unicode(self):
         return self.space.fsdecode_w(self.w_obj)
@@ -85,7 +85,7 @@
             fname = FileEncoder(space, w_fname)
             return func(fname, *args)
         else:
-            fname = space.fsencode_w(w_fname)
+            fname = space.bytes0_w(w_fname)
             return func(fname, *args)
     return dispatch
 
@@ -148,11 +148,15 @@
     try:
         path_b = space.fsencode_w(w_value)
         return Path(-1, path_b, None, w_value)
-    except OperationError:
+    except OperationError as e:
+        if not e.match(space, space.w_TypeError):
+            raise
         if allow_fd:
             fd = unwrap_fd(space, w_value, "string, bytes or integer")
             return Path(fd, None, None, w_value)
-    raise oefmt(space.w_TypeError, "illegal type for path parameter")
+    raise oefmt(space.w_TypeError,
+                "illegal type for path parameter (expected "
+                "string or bytes, got %T)", w_value)
 
 class _PathOrFd(Unwrapper):
     def unwrap(self, space, w_value):
@@ -226,16 +230,22 @@
   If it is unavailable, using it will raise a NotImplementedError."""
     if rposix.O_CLOEXEC is not None:
         flags |= rposix.O_CLOEXEC
+    while True:
+        try:
+            if rposix.HAVE_OPENAT and dir_fd != DEFAULT_DIR_FD:
+                path = space.fsencode_w(w_path)
+                fd = rposix.openat(path, flags, mode, dir_fd)
+            else:
+                fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
+            break
+        except OSError as e:
+            wrap_oserror2(space, e, w_path, eintr_retry=True)
     try:
-        if rposix.HAVE_OPENAT and dir_fd != DEFAULT_DIR_FD:
-            path = space.fsencode_w(w_path)
-            fd = rposix.openat(path, flags, mode, dir_fd)
-        else:
-            fd = dispatch_filename(rposix.open)(space, w_path, flags, mode)
         _open_inhcache.set_non_inheritable(fd)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
-    return space.wrap(fd)
+        rposix.c_close(fd)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
+    return space.newint(fd)
 
 @unwrap_spec(fd=c_int, position=r_longlong, how=c_int)
 def lseek(space, fd, position, how):
@@ -245,9 +255,9 @@
     try:
         pos = os.lseek(fd, position, how)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(pos)
+        return space.newint(pos)
 
 @unwrap_spec(fd=c_int)
 def isatty(space, fd):
@@ -256,19 +266,20 @@
     try:
         res = os.isatty(fd)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(res)
+        return space.newbool(res)
 
 @unwrap_spec(fd=c_int, length=int)
 def read(space, fd, length):
     """Read data from a file descriptor."""
-    try:
-        s = os.read(fd, length)
-    except OSError as e:
-        raise wrap_oserror(space, e)
-    else:
-        return space.newbytes(s)
+    while True:
+        try:
+            s = os.read(fd, length)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+            return space.newbytes(s)
         
 @unwrap_spec(fd=c_int)
 def write(space, fd, w_data):
@@ -284,15 +295,19 @@
 
 @unwrap_spec(fd=c_int, length=int, offset=int)
 def pread(space, fd, length, offset):
+    """Read a string to a file descriptor at a given offset.
+    """
     try:
         s = rposix.pread(fd, length, offset)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=True)
     else:
-        return space.newbytes(s)
+       return space.newbytes(s)
 
 @unwrap_spec(fd=c_int, offset=int)
 def pwrite(space, fd, w_data, offset):
+    """Write a string to a file descriptor at a given offset.
+    """
     data = space.getarg_w('y*', w_data)
     try:
         res = rposix.pwrite(fd, data.as_str(), offset)
@@ -304,10 +319,14 @@
 @unwrap_spec(fd=c_int)
 def close(space, fd):
     """Close a file descriptor (for low level IO)."""
+    # PEP 475 note: os.close() must not retry upon EINTR.  Like in
+    # previous versions of Python it raises OSError in this case.
+    # The text of PEP 475 seems to suggest that EINTR is eaten and
+    # hidden from app-level, but it is not the case in CPython 3.5.2.
     try:
         os.close(fd)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(fd_low=c_int, fd_high=c_int)
 def closerange(fd_low, fd_high):
@@ -317,10 +336,12 @@
 @unwrap_spec(fd=c_int, length=r_longlong)
 def ftruncate(space, fd, length):
     """Truncate a file (by file descriptor) to a specified length."""
-    try:
-        os.ftruncate(fd, length)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    while True:
+        try:
+            os.ftruncate(fd, length)
+            break
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
 
 def truncate(space, w_path, w_length):
     """Truncate a file to a specified length."""
@@ -330,7 +351,7 @@
         if space.isinstance_w(w_path, space.w_int):
             w_fd = w_path
         else:
-            w_fd = open(space, w_path, os.O_RDWR | os.O_CREAT)
+            w_fd = open(space, w_path, os.O_WRONLY)
             allocated_fd = True
 
         fd = space.c_filedescriptor_w(w_fd)
@@ -344,19 +365,23 @@
 def fsync(space, w_fd):
     """Force write of file with filedescriptor to disk."""
     fd = space.c_filedescriptor_w(w_fd)
-    try:
-        os.fsync(fd)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    while True:
+        try:
+            os.fsync(fd)
+            break
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
 
 def fdatasync(space, w_fd):
     """Force write of file with filedescriptor to disk.
 Does not force update of metadata."""
     fd = space.c_filedescriptor_w(w_fd)
-    try:
-        os.fdatasync(fd)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    while True:
+        try:
+            os.fdatasync(fd)
+            break
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
 
 def sync(space):
     """Force write of everything to disk."""
@@ -366,10 +391,12 @@
     """Change to the directory of the given file descriptor.  fildes must be
 opened on a directory, not a file."""
     fd = space.c_filedescriptor_w(w_fd)
-    try:
-        os.fchdir(fd)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    while True:
+        try:
+            os.fchdir(fd)
+            break
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
 
 # ____________________________________________________________
 
@@ -386,7 +413,7 @@
         if i < rposix_stat.N_INDEXABLE_FIELDS:
             # get the first 10 items by indexing; this gives us
             # 'st_Xtime' as an integer, too
-            w_value = space.wrap(st[i])
+            w_value = space.newint(st[i])
             lst[i] = w_value
         else:
             try:
@@ -396,8 +423,8 @@
                 assert name.startswith('nsec_')
                 value = rposix_stat.get_stat_ns_as_bigint(st, name[5:])
                 value = value.tolong() % 1000000000
-            w_value = space.wrap(value)
-            space.setitem(w_keywords, space.wrap(name), w_value)
+            w_value = space.newint(value)
+            space.setitem(w_keywords, space.newtext(name), w_value)
 
     # Note: 'w_keywords' contains the three attributes 'nsec_Xtime'.
     # We have an app-level property in app_posix.stat_result to
@@ -406,27 +433,27 @@
     # non-rounded values for name-based access
     if stat_float_times:
         space.setitem(w_keywords,
-                      space.wrap('st_atime'), space.wrap(st.st_atime))
+                      space.newtext('st_atime'), space.newfloat(st.st_atime))
         space.setitem(w_keywords,
-                      space.wrap('st_mtime'), space.wrap(st.st_mtime))
+                      space.newtext('st_mtime'), space.newfloat(st.st_mtime))
         space.setitem(w_keywords,
-                      space.wrap('st_ctime'), space.wrap(st.st_ctime))
+                      space.newtext('st_ctime'), space.newfloat(st.st_ctime))
     #else:
     #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
-                                  space.wrap('stat_result'))
+                                  space.newtext('stat_result'))
     return space.call_function(w_stat_result, w_tuple, w_keywords)
 
 
 def build_statvfs_result(space, st):
     vals_w = [None] * len(rposix_stat.STATVFS_FIELDS)
     for i, (name, _) in STATVFS_FIELDS:
-        vals_w[i] = space.wrap(getattr(st, name))
+        vals_w[i] = space.newint(getattr(st, name))
     w_tuple = space.newtuple(vals_w)
     w_statvfs_result = space.getattr(
-        space.getbuiltinmodule(os.name), space.wrap('statvfs_result'))
+        space.getbuiltinmodule(os.name), space.newtext('statvfs_result'))
     return space.call_function(w_statvfs_result, w_tuple)
 
 
@@ -434,12 +461,13 @@
 def fstat(space, fd):
     """Perform a stat system call on the file referenced to by an open
 file descriptor."""
-    try:
-        st = rposix_stat.fstat(fd)
-    except OSError as e:
-        raise wrap_oserror(space, e)
-    else:
-        return build_stat_result(space, st)
+    while True:
+        try:
+            st = rposix_stat.fstat(fd)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+            return build_stat_result(space, st)
 
 @unwrap_spec(
     path=path_or_fd(allow_fd=True),
@@ -485,7 +513,7 @@
             raise oefmt(space.w_NotImplementedError,
                 "%s: unsupported argument combination", funcname)
     except OSError as e:
-        raise wrap_oserror2(space, e, path.w_path)
+        raise wrap_oserror2(space, e, path.w_path, eintr_retry=False)
     else:
         return build_stat_result(space, st)
 
@@ -515,19 +543,20 @@
     state = space.fromcache(StatState)
 
     if newval == -1:
-        return space.wrap(state.stat_float_times)
+        return space.newbool(state.stat_float_times)
     else:
         state.stat_float_times = (newval != 0)
 
 
 @unwrap_spec(fd=c_int)
 def fstatvfs(space, fd):
-    try:
-        st = rposix_stat.fstatvfs(fd)
-    except OSError as e:
-        raise wrap_oserror(space, e)
-    else:
-        return build_statvfs_result(space, st)
+    while True:
+        try:
+            st = rposix_stat.fstatvfs(fd)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+            return build_statvfs_result(space, st)
 
 
 def statvfs(space, w_path):
@@ -543,7 +572,7 @@
             rposix_stat.statvfs,
             allow_fd_fn=rposix_stat.fstatvfs)(space, w_path)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
     else:
         return build_statvfs_result(space, st)
 
@@ -555,17 +584,19 @@
     try:
         newfd = rposix.dup(fd, inheritable=False)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(newfd)
+        return space.newint(newfd)
 
 @unwrap_spec(fd=c_int, fd2=c_int, inheritable=bool)
 def dup2(space, fd, fd2, inheritable=1):
     """Duplicate a file descriptor."""
+    # like os.close(), this can still raise EINTR to app-level in
+    # CPython 3.5.2
     try:
         rposix.dup2(fd, fd2, inheritable)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(mode=c_int,
     dir_fd=DirFD(rposix.HAVE_FACCESSAT), effective_ids=bool,
@@ -610,9 +641,9 @@
         else:
             ok = dispatch_filename(rposix.access)(space, w_path, mode)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
     else:
-        return space.wrap(ok)
+        return space.newbool(ok)
 
 
 def times(space):
@@ -624,13 +655,13 @@
     try:
         times = os.times()
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.newtuple([space.wrap(times[0]),
-                               space.wrap(times[1]),
-                               space.wrap(times[2]),
-                               space.wrap(times[3]),
-                               space.wrap(times[4])])
+        return space.newtuple([space.newfloat(times[0]),
+                               space.newfloat(times[1]),
+                               space.newfloat(times[2]),
+                               space.newfloat(times[3]),
+                               space.newfloat(times[4])])
 
 @unwrap_spec(command='fsencode')
 def system(space, command):
@@ -638,9 +669,9 @@
     try:
         rc = os.system(command)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     else:
-        return space.wrap(rc)
+        return space.newint(rc)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
 def unlink(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
@@ -659,7 +690,7 @@
         else:
             dispatch_filename(rposix.unlink)(space, w_path)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
 def remove(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
@@ -678,7 +709,7 @@
         else:
             dispatch_filename(rposix.unlink)(space, w_path)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
 
 def _getfullpathname(space, w_path):
     """helper for ntpath.abspath """
@@ -686,13 +717,13 @@
         if space.isinstance_w(w_path, space.w_unicode):
             path = FileEncoder(space, w_path)
             fullpath = rposix.getfullpathname(path)
-            w_fullpath = space.wrap(fullpath)
+            w_fullpath = space.newunicode(fullpath)
         else:
-            path = space.str0_w(w_path)
+            path = space.bytes0_w(w_path)
             fullpath = rposix.getfullpathname(path)
             w_fullpath = space.newbytes(fullpath)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
     else:
         return w_fullpath
 
@@ -701,7 +732,7 @@
     try:
         cur = os.getcwd()
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     else:
         return space.newbytes(cur)
 
@@ -711,9 +742,9 @@
         try:
             cur = os.getcwdu()
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
         else:
-            return space.wrap(cur)
+            return space.newunicode(cur)
 else:
     def getcwd(space):
         """Return the current working directory as a string."""
@@ -728,7 +759,7 @@
         else:
             dispatch_filename(rposix.chdir)(space, w_path)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKDIRAT))
 def mkdir(space, w_path, mode=0o777, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
@@ -749,7 +780,7 @@
         else:
             dispatch_filename(rposix.mkdir)(space, w_path, mode)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_UNLINKAT))
 def rmdir(space, w_path, __kwonly__, dir_fd=DEFAULT_DIR_FD):
@@ -768,13 +799,13 @@
         else:
             dispatch_filename(rposix.rmdir)(space, w_path)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
 
 @unwrap_spec(code=c_int)
 def strerror(space, code):
     """Translate an error code to a message string."""
     try:
-        return space.wrap(_strerror(code))
+        return space.newunicode(_strerror(code))
     except ValueError:
         raise oefmt(space.w_ValueError, "strerror() argument out of range")
 
@@ -783,15 +814,15 @@
     try:
         cur = os.getlogin()
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap_fsdecoded(cur)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newfilename(cur)
 
 # ____________________________________________________________
 
 def getstatfields(space):
     # for app_posix.py: export the list of 'st_xxx' names that we know
     # about at RPython level
-    return space.newlist([space.wrap(name) for _, (name, _) in STAT_FIELDS])
+    return space.newlist([space.newtext(name) for _, (name, _) in STAT_FIELDS])
 
 
 class State:
@@ -823,7 +854,7 @@
         for key, value in rwin32._wenviron_items():
             if isinstance(key, str):
                 key = key.upper()
-            space.setitem(w_env, space.wrap(key), space.wrap(value))
+            space.setitem(w_env, space.newtext(key), space.newtext(value))
 
     @unwrap_spec(name=unicode, value=unicode)
     def putenv(space, name, value):
@@ -836,7 +867,7 @@
         try:
             rwin32._wputenv(name, value)
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
 else:
     def _convertenviron(space, w_env):
         for key, value in os.environ.items():
@@ -847,7 +878,7 @@
         try:
             dispatch_filename_2(rposix.putenv)(space, w_name, w_value)
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
 
     def unsetenv(space, w_name):
         """Delete an environment variable."""
@@ -856,7 +887,7 @@
         except KeyError:
             pass
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
 
 
 def listdir(space, w_path=None):
@@ -875,11 +906,11 @@
     if space.is_none(w_path):
         w_path = space.newunicode(u".")
     if space.isinstance_w(w_path, space.w_bytes):
-        dirname = space.str0_w(w_path)
+        dirname = space.bytes0_w(w_path)
         try:
             result = rposix.listdir(dirname)
         except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
+            raise wrap_oserror2(space, e, w_path, eintr_retry=False)
         return space.newlist_bytes(result)
     try:
         path = space.fsencode_w(w_path)
@@ -893,35 +924,35 @@
         try:
             result = rposix.fdlistdir(os.dup(fd))
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
     else:
         dirname = FileEncoder(space, w_path)
         try:
             result = rposix.listdir(dirname)
         except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
+            raise wrap_oserror2(space, e, w_path, eintr_retry=False)
     len_result = len(result)
     result_w = [None] * len_result
     for i in range(len_result):
         if _WIN32:
-            result_w[i] = space.wrap(result[i])
+            result_w[i] = space.newunicode(result[i])
         else:
-            result_w[i] = space.wrap_fsdecoded(result[i])
+            result_w[i] = space.newfilename(result[i])
     return space.newlist(result_w)
 
 @unwrap_spec(fd=c_int)
 def get_inheritable(space, fd):
     try:
-        return space.wrap(rposix.get_inheritable(fd))
+        return space.newbool(rposix.get_inheritable(fd))
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(fd=c_int, inheritable=int)
 def set_inheritable(space, fd, inheritable):
     try:
         rposix.set_inheritable(fd, inheritable)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 _pipe_inhcache = rposix.SetNonInheritableCache()
 
@@ -929,19 +960,24 @@
     "Create a pipe.  Returns (read_end, write_end)."
     try:
         fd1, fd2 = rposix.pipe(rposix.O_CLOEXEC or 0)
+    except OSError as e:
+        raise wrap_oserror(space, e, eintr_retry=False)
+    try:
         _pipe_inhcache.set_non_inheritable(fd1)
         _pipe_inhcache.set_non_inheritable(fd2)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+        rposix.c_close(fd2)
+        rposix.c_close(fd1)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newtuple([space.newint(fd1), space.newint(fd2)])
 
 @unwrap_spec(flags=c_int)
 def pipe2(space, flags):
     try:
         fd1, fd2 = rposix.pipe2(flags)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newtuple([space.newint(fd1), space.newint(fd2)])
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_FCHMODAT),
              follow_symlinks=bool)
@@ -966,11 +1002,12 @@
     if not rposix.HAVE_FCHMODAT:
         if not follow_symlinks:
             raise argument_unavailable(space, "chmod", "follow_symlinks")
-        try:
-            dispatch_filename(rposix.chmod)(space, w_path, mode)
-            return
-        except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
+        while True:
+            try:
+                dispatch_filename(rposix.chmod)(space, w_path, mode)
+                return
+            except OSError as e:
+                wrap_oserror2(space, e, w_path, eintr_retry=True)
 
     try:
         path = space.fsencode_w(w_path)
@@ -979,16 +1016,25 @@
             raise oefmt(space.w_TypeError,
                 "argument should be string, bytes or integer, not %T", w_path)
         fd = unwrap_fd(space, w_path)
-        _chmod_fd(space, fd, mode)
-    else:
+        # NB. in CPython 3.5.2, os.chmod(fd) propagates EINTR to app-level,
+        # but os.fchmod(fd) retries automatically.  This might be fixed in
+        # more recent CPythons.
+        while True:
+            try:
+                os.fchmod(fd, mode)
+                return
+            except OSError as e:
+                wrap_oserror(space, e, eintr_retry=True)
+    while True:
         try:
             _chmod_path(path, mode, dir_fd, follow_symlinks)
+            break
         except OSError as e:
             if not follow_symlinks and e.errno in (ENOTSUP, EOPNOTSUPP):
                 # fchmodat() doesn't actually implement follow_symlinks=False
                 # so raise NotImplementedError in this case
                 raise argument_unavailable(space, "chmod", "follow_symlinks")
-            raise wrap_oserror2(space, e, w_path)
+            wrap_oserror2(space, e, w_path, eintr_retry=True)
 
 def _chmod_path(path, mode, dir_fd, follow_symlinks):
     if dir_fd != DEFAULT_DIR_FD or not follow_symlinks:
@@ -996,19 +1042,17 @@
     else:
         rposix.chmod(path, mode)
 
-def _chmod_fd(space, fd, mode):
-    try:
-        os.fchmod(fd, mode)
-    except OSError as e:
-        raise wrap_oserror(space, e)
-
-
 @unwrap_spec(fd=c_int, mode=c_int)
 def fchmod(space, fd, mode):
     """\
     Change the access permissions of the file given by file descriptor fd.
     """
-    _chmod_fd(space, fd, mode)
+    while True:
+        try:
+            os.fchmod(fd, mode)
+            break
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
@@ -1032,7 +1076,8 @@
         else:
             dispatch_filename_2(rposix.rename)(space, w_src, w_dst)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst,
+                            eintr_retry=False)
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
@@ -1056,7 +1101,8 @@
         else:
             dispatch_filename_2(rposix.replace)(space, w_src, w_dst)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst,
+                            eintr_retry=False)
 
 @unwrap_spec(mode=c_int, dir_fd=DirFD(rposix.HAVE_MKFIFOAT))
 def mkfifo(space, w_path, mode=0666, __kwonly__=None, dir_fd=DEFAULT_DIR_FD):
@@ -1068,14 +1114,18 @@
   and path should be relative; path will then be relative to that directory.
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
-    try:
-        if rposix.HAVE_MKFIFOAT and dir_fd != DEFAULT_DIR_FD:
-            path = space.fsencode_w(w_path)
-            rposix.mkfifoat(path, mode, dir_fd)
-        else:
-            dispatch_filename(rposix.mkfifo)(space, w_path, mode)
-    except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+    # CPython 3.5.2: why does os.mkfifo() retry automatically if it
+    # gets EINTR, but not os.mkdir()?
+    while True:
+        try:
+            if rposix.HAVE_MKFIFOAT and dir_fd != DEFAULT_DIR_FD:
+                path = space.fsencode_w(w_path)
+                rposix.mkfifoat(path, mode, dir_fd)
+            else:
+                dispatch_filename(rposix.mkfifo)(space, w_path, mode)
+            break
+        except OSError as e:
+            wrap_oserror2(space, e, w_path, eintr_retry=True)
 
 @unwrap_spec(mode=c_int, device=c_int, dir_fd=DirFD(rposix.HAVE_MKNODAT))
 def mknod(space, w_path, mode=0600, device=0,
@@ -1093,28 +1143,30 @@
   and path should be relative; path will then be relative to that directory.
 dir_fd may not be implemented on your platform.
   If it is unavailable, using it will raise a NotImplementedError."""
-    try:
-        if rposix.HAVE_MKNODAT and dir_fd != DEFAULT_DIR_FD:
-            fname = space.fsencode_w(w_path)
-            rposix.mknodat(fname, mode, device, dir_fd)
-        else:
-            dispatch_filename(rposix.mknod)(space, w_path, mode, device)
-    except OSError as e:
-        raise wrap_oserror2(space, e, w_path)
+    while True:
+        try:
+            if rposix.HAVE_MKNODAT and dir_fd != DEFAULT_DIR_FD:
+                fname = space.fsencode_w(w_path)
+                rposix.mknodat(fname, mode, device, dir_fd)
+            else:
+                dispatch_filename(rposix.mknod)(space, w_path, mode, device)
+            break
+        except OSError as e:
+            wrap_oserror2(space, e, w_path, eintr_retry=True)
 
 @unwrap_spec(mask=c_int)
 def umask(space, mask):
     "Set the current numeric umask and return the previous umask."
     prevmask = os.umask(mask)
-    return space.wrap(prevmask)
+    return space.newint(prevmask)
 
 def getpid(space):
     "Return the current process id."
     try:
         pid = os.getpid()
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(pid)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(pid)
 
 @unwrap_spec(pid=c_int, signal=c_int)
 def kill(space, pid, signal):
@@ -1122,7 +1174,7 @@
     try:
         rposix.kill(pid, signal)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(pgid=c_int, signal=c_int)
 def killpg(space, pgid, signal):
@@ -1130,7 +1182,7 @@
     try:
         os.killpg(pgid, signal)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 def abort(space):
     """Abort the interpreter immediately.  This 'dumps core' or otherwise fails
@@ -1139,11 +1191,9 @@
     rposix.kill(os.getpid(), signal.SIGABRT)
 
 @unwrap_spec(
-    src='fsencode', dst='fsencode',  # <- simpler: link() is never on Windows
     src_dir_fd=DirFD(rposix.HAVE_LINKAT), dst_dir_fd=DirFD(rposix.HAVE_LINKAT),
     follow_symlinks=bool)
-def link(
-        space, src, dst, __kwonly__,
+def link(space, w_src, w_dst, __kwonly__,
         src_dir_fd=DEFAULT_DIR_FD, dst_dir_fd=DEFAULT_DIR_FD,
         follow_symlinks=True):
     """\
@@ -1160,6 +1210,8 @@
 src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your
   platform.  If they are unavailable, using them will raise a
   NotImplementedError."""
+    src = space.fsencode_w(w_src)
+    dst = space.fsencode_w(w_dst)
     try:
         if (rposix.HAVE_LINKAT and
             (src_dir_fd != DEFAULT_DIR_FD or dst_dir_fd != DEFAULT_DIR_FD
@@ -1168,7 +1220,8 @@
         else:
             rposix.link(src, dst)
     except OSError as e:
-        raise wrap_oserror(space, e, filename=src, filename2=dst)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst,
+                            eintr_retry=False)
 
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_SYMLINKAT))
@@ -1195,7 +1248,8 @@
         else:
             dispatch_filename_2(rposix.symlink)(space, w_src, w_dst)
     except OSError as e:
-        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst,
+                            eintr_retry=False)
 
 
 @unwrap_spec(
@@ -1216,7 +1270,7 @@
         else:
             result = call_rposix(rposix.readlink, path)
     except OSError as e:
-        raise wrap_oserror2(space, e, path.w_path)
+        raise wrap_oserror2(space, e, path.w_path, eintr_retry=False)
     w_result = space.newbytes(result)
     if space.isinstance_w(path.w_path, space.w_unicode):
         return space.fsdecode(w_result)
@@ -1264,7 +1318,7 @@
         except:
             # Don't clobber the OSError if the fork failed
             pass
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     if pid == 0:
         run_fork_hooks('child', space)
     else:
@@ -1273,22 +1327,27 @@
 
 def fork(space):
     pid, irrelevant = _run_forking_function(space, "F")
-    return space.wrap(pid)
+    return space.newint(pid)
 
 def openpty(space):
     "Open a pseudo-terminal, returning open fd's for both master and slave end."
+    master_fd = slave_fd = -1
     try:
         master_fd, slave_fd = os.openpty()
         rposix.set_inheritable(master_fd, False)
         rposix.set_inheritable(slave_fd, False)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)])
+        if master_fd >= 0:
+            rposix.c_close(master_fd)
+        if slave_fd >= 0:
+            rposix.c_close(slave_fd)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newtuple([space.newint(master_fd), space.newint(slave_fd)])
 
 def forkpty(space):
     pid, master_fd = _run_forking_function(space, "P")
-    return space.newtuple([space.wrap(pid),
-                           space.wrap(master_fd)])
+    return space.newtuple([space.newint(pid),
+                           space.newint(master_fd)])
 
 @unwrap_spec(pid=c_int, options=c_int)
 def waitpid(space, pid, options):
@@ -1296,11 +1355,15 @@
 
     Wait for completion of a given child process.
     """
-    try:
-        pid, status = os.waitpid(pid, options)
-    except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(pid), space.wrap(status)])
+    while True:
+        try:
+            pid, status = os.waitpid(pid, options)
+            break
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+    return space.newtuple([space.newint(pid), space.newint(status)])
+
+# missing: waitid()
 
 @unwrap_spec(status=c_int)
 def _exit(space, status):
@@ -1329,7 +1392,7 @@
     try:
         os.execv(command, args)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 
 def _env2interp(space, w_env):
@@ -1374,12 +1437,12 @@
         try:
             rposix.fexecve(fd, args, env)
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
     else:
         try:
             os.execve(path, args, env)
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(mode=int, path='fsencode')
 def spawnv(space, mode, path, w_argv):
@@ -1387,8 +1450,8 @@
     try:
         ret = os.spawnv(mode, path, args)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(ret)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(ret)
 
 @unwrap_spec(mode=int, path='fsencode')
 def spawnve(space, mode, path, w_argv, w_env):
@@ -1397,8 +1460,8 @@
     try:
         ret = os.spawnve(mode, path, args, env)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(ret)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(ret)
 
 
 @unwrap_spec(
@@ -1505,7 +1568,7 @@
         #    something is wrong with the file, when it also
         #    could be the time stamp that gives a problem. */
         # so we use wrap_oserror() instead of wrap_oserror2() here
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @specialize.arg(1)
 def do_utimes(space, func, arg, utime):
@@ -1522,7 +1585,7 @@
             func(arg, (atime, mtime))
     except OSError as e:
         # see comment above: don't use wrap_oserror2()
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @specialize.argtype(1)
 def _dispatch_utime(path, times):
@@ -1549,7 +1612,7 @@
         return time, 0
 
 def convert_ns(space, w_ns_time):
-    w_billion = space.wrap(1000000000)
+    w_billion = space.newint(1000000000)
     w_res = space.divmod(w_ns_time, w_billion)
     res_w = space.fixedview(w_res)
     time_int = space.int_w(res_w[0])
@@ -1565,12 +1628,12 @@
     try:
         r = os.uname()
     except OSError as e:
-        raise wrap_oserror(space, e)
-    l_w = [space.wrap_fsdecoded(i)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    l_w = [space.newfilename(i)
            for i in [r[0], r[1], r[2], r[3], r[4]]]
     w_tuple = space.newtuple(l_w)
     w_uname_result = space.getattr(space.getbuiltinmodule(os.name),
-                                   space.wrap('uname_result'))
+                                   space.newtext('uname_result'))
     return space.call_function(w_uname_result, w_tuple)
 
 def getuid(space):
@@ -1589,7 +1652,7 @@
     try:
         os.setuid(uid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(euid=c_uid_t)
 def seteuid(space, euid):
@@ -1600,7 +1663,7 @@
     try:
         os.seteuid(euid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(gid=c_gid_t)
 def setgid(space, gid):
@@ -1611,7 +1674,7 @@
     try:
         os.setgid(gid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(egid=c_gid_t)
 def setegid(space, egid):
@@ -1622,18 +1685,18 @@
     try:
         os.setegid(egid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
- at unwrap_spec(path='fsencode')
-def chroot(space, path):
+def chroot(space, w_path):
     """ chroot(path)
 
     Change root directory to path.
     """
+    path = space.fsencode_w(w_path)
     try:
         os.chroot(path)
     except OSError as e:
-        raise wrap_oserror(space, e, path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
     return space.w_None
 
 def getgid(space):
@@ -1665,7 +1728,7 @@
     try:
         list = os.getgroups()
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     return space.newlist([wrap_gid(space, e) for e in list])
 
 def setgroups(space, w_groups):
@@ -1679,9 +1742,9 @@
     try:
         os.setgroups(list[:])
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
- at unwrap_spec(username=str, gid=c_gid_t)
+ at unwrap_spec(username='text', gid=c_gid_t)
 def initgroups(space, username, gid):
     """ initgroups(username, gid) -> None
 
@@ -1692,14 +1755,14 @@
     try:
         os.initgroups(username, gid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 def getpgrp(space):
     """ getpgrp() -> pgrp
 
     Return the current process group id.
     """
-    return space.wrap(os.getpgrp())
+    return space.newint(os.getpgrp())
 
 def setpgrp(space):
     """ setpgrp()
@@ -1709,7 +1772,7 @@
     try:
         os.setpgrp()
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     return space.w_None
 
 def getppid(space):
@@ -1717,7 +1780,7 @@
 
     Return the parent's process id.
     """
-    return space.wrap(os.getppid())
+    return space.newint(os.getppid())
 
 @unwrap_spec(pid=c_int)
 def getpgid(space, pid):
@@ -1728,8 +1791,8 @@
     try:
         pgid = os.getpgid(pid)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(pgid)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(pgid)
 
 @unwrap_spec(pid=c_int, pgrp=c_int)
 def setpgid(space, pid, pgrp):
@@ -1740,7 +1803,7 @@
     try:
         os.setpgid(pid, pgrp)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     return space.w_None
 
 @unwrap_spec(ruid=c_uid_t, euid=c_uid_t)
@@ -1752,7 +1815,7 @@
     try:
         os.setreuid(ruid, euid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(rgid=c_gid_t, egid=c_gid_t)
 def setregid(space, rgid, egid):
@@ -1763,7 +1826,7 @@
     try:
         os.setregid(rgid, egid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(pid=c_int)
 def getsid(space, pid):
@@ -1774,8 +1837,8 @@
     try:
         sid = os.getsid(pid)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(sid)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(sid)
 
 def setsid(space):
     """ setsid()
@@ -1785,7 +1848,7 @@
     try:
         os.setsid()
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     return space.w_None
 
 @unwrap_spec(fd=c_int)
@@ -1797,8 +1860,8 @@
     try:
         pgid = os.tcgetpgrp(fd)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(pgid)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(pgid)
 
 @unwrap_spec(fd=c_int, pgid=c_gid_t)
 def tcsetpgrp(space, fd, pgid):
@@ -1809,7 +1872,7 @@
     try:
         os.tcsetpgrp(fd, pgid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 def getresuid(space):
     """ getresuid() -> (ruid, euid, suid)
@@ -1819,7 +1882,7 @@
     try:
         (ruid, euid, suid) = os.getresuid()
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     return space.newtuple([wrap_uid(space, ruid),
                            wrap_uid(space, euid),
                            wrap_uid(space, suid)])
@@ -1832,7 +1895,7 @@
     try:
         (rgid, egid, sgid) = os.getresgid()
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     return space.newtuple([wrap_gid(space, rgid),
                            wrap_gid(space, egid),
                            wrap_gid(space, sgid)])
@@ -1846,7 +1909,7 @@
     try:
         os.setresuid(ruid, euid, suid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 @unwrap_spec(rgid=c_gid_t, egid=c_gid_t, sgid=c_gid_t)
 def setresgid(space, rgid, egid, sgid):
@@ -1857,13 +1920,36 @@
     try:
         os.setresgid(rgid, egid, sgid)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
+
+ at unwrap_spec(which=int, who=int)
+def getpriority(space, which, who):
+    """ getpriority(which, who) -> int
+
+    Get program scheduling priority.
+    """
+    try:
+        returned_priority = rposix.getpriority(which, who)
+    except OSError as e:
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(returned_priority)
+
+ at unwrap_spec(which=int, who=int, priority=int)
+def setpriority(space, which, who, priority):
+    """ setpriority(which, who, priority)
+
+    Set program scheduling priority.
+    """
+    try:
+        rposix.setpriority(which, who, priority)
+    except OSError as e:
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 def declare_new_w_star(name):
     if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
-            return space.wrap(getattr(os, name)(status))
+            return space.newint(getattr(os, name)(status))
     else:
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
@@ -1881,16 +1967,16 @@
 @unwrap_spec(fd=c_int)
 def ttyname(space, fd):
     try:
-        return space.wrap_fsdecoded(os.ttyname(fd))
+        return space.newfilename(os.ttyname(fd))
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 
 def confname_w(space, w_name, namespace):
     # XXX slightly non-nice, reuses the sysconf of the underlying os module
     if space.isinstance_w(w_name, space.w_unicode):
         try:
-            num = namespace[space.str_w(w_name)]
+            num = namespace[space.text_w(w_name)]
         except KeyError:
             raise oefmt(space.w_ValueError, "unrecognized configuration name")
     else:
@@ -1902,8 +1988,8 @@
     try:
         res = os.sysconf(num)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(res)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(res)
 
 @unwrap_spec(fd=c_int)
 def fpathconf(space, fd, w_name):
@@ -1911,8 +1997,8 @@
     try:
         res = os.fpathconf(fd, num)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(res)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(res)
 
 @unwrap_spec(path=path_or_fd(allow_fd=hasattr(os, 'fpathconf')))
 def pathconf(space, path, w_name):
@@ -1921,21 +2007,21 @@
         try:
             res = os.fpathconf(path.as_fd, num)
         except OSError as e:
-            raise wrap_oserror(space, e)
+            raise wrap_oserror(space, e, eintr_retry=False)
     else:
         try:
             res = os.pathconf(path.as_bytes, num)
         except OSError as e:
-            raise wrap_oserror2(space, e, path.w_path)
-    return space.wrap(res)
+            raise wrap_oserror2(space, e, path.w_path, eintr_retry=False)
+    return space.newint(res)
 
 def confstr(space, w_name):
     num = confname_w(space, w_name, os.confstr_names)
     try:
         res = os.confstr(num)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(res)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newtext(res)
 
 @unwrap_spec(
     uid=c_uid_t, gid=c_gid_t,
@@ -1975,11 +2061,16 @@
         if not follow_symlinks:
             raise oefmt(space.w_ValueError,
                 "chown: cannnot use fd and follow_symlinks together")
-        try:
-            os.fchown(fd, uid, gid)
-        except OSError as e:
-            raise wrap_oserror(space, e)
-    else:
+        # NB. in CPython 3.5.2, os.chown(fd) propagates EINTR to app-level,
+        # but os.fchown(fd) retries automatically.  This might be fixed in
+        # more recent CPythons.
+        while True:
+            try:
+                os.fchown(fd, uid, gid)
+                return
+            except OSError as e:
+                wrap_oserror(space, e, eintr_retry=True)
+    while True:
         # String case
         try:
             if (rposix.HAVE_LCHOWN and
@@ -1992,21 +2083,23 @@
                 assert follow_symlinks
                 assert dir_fd == DEFAULT_DIR_FD
                 os.chown(path, uid, gid)
+            break
         except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
+            wrap_oserror2(space, e, w_path, eintr_retry=True)
 
 
- at unwrap_spec(path='fsencode', uid=c_uid_t, gid=c_gid_t)
-def lchown(space, path, uid, gid):
+ at unwrap_spec(uid=c_uid_t, gid=c_gid_t)
+def lchown(space, w_path, uid, gid):
     """lchown(path, uid, gid)
 
 Change the owner and group id of path to the numeric uid and gid.
 This function will not follow symbolic links.
 Equivalent to os.chown(path, uid, gid, follow_symlinks=False)."""
+    path = space.fsencode_w(w_path)
     try:
         os.lchown(path, uid, gid)
     except OSError as e:
-        raise wrap_oserror(space, e, path)
+        raise wrap_oserror2(space, e, w_path, eintr_retry=False)
 
 @unwrap_spec(uid=c_uid_t, gid=c_gid_t)
 def fchown(space, w_fd, uid, gid):
@@ -2015,34 +2108,36 @@
 Change the owner and group id of the file given by file descriptor
 fd to the numeric uid and gid.  Equivalent to os.chown(fd, uid, gid)."""
     fd = space.c_filedescriptor_w(w_fd)
-    try:
-        os.fchown(fd, uid, gid)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    while True:
+        try:
+            os.fchown(fd, uid, gid)
+            break
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
 
 def getloadavg(space):
     try:
         load = os.getloadavg()
     except OSError:
         raise oefmt(space.w_OSError, "Load averages are unobtainable")
-    return space.newtuple([space.wrap(load[0]),
-                           space.wrap(load[1]),
-                           space.wrap(load[2])])
+    return space.newtuple([space.newfloat(load[0]),
+                           space.newfloat(load[1]),
+                           space.newfloat(load[2])])
 
 @unwrap_spec(major=c_int, minor=c_int)
 def makedev(space, major, minor):
     result = os.makedev(major, minor)
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(device="c_uint")
 def major(space, device):
     result = os.major(intmask(device))
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(device="c_uint")
 def minor(space, device):
     result = os.minor(intmask(device))
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(increment=c_int)
 def nice(space, increment):
@@ -2051,8 +2146,14 @@
     try:
         res = os.nice(increment)
     except OSError as e:
-        raise wrap_oserror(space, e)
-    return space.wrap(res)
+        raise wrap_oserror(space, e, eintr_retry=False)
+    return space.newint(res)
+
+class SigCheck:
+    pass
+_sigcheck = SigCheck()
+def _signal_checker():
+    _sigcheck.space.getexecutioncontext().checksignals()
 
 @unwrap_spec(size=int)
 def urandom(space, size):
@@ -2062,16 +2163,22 @@
     """
     context = get(space).random_context
     try:
-        return space.newbytes(rurandom.urandom(context, size))
+        # urandom() takes a final argument that should be a regular function,
+        # not a bound method like 'getexecutioncontext().checksignals'.
+        # Otherwise, we can't use it from several independent places.
+        _sigcheck.space = space
+        return space.newbytes(rurandom.urandom(context, n, _signal_checker))
     except OSError as e:
-        raise wrap_oserror(space, e)
+        # 'rurandom' should catch and retry internally if it gets EINTR
+        # (at least in os.read(), which is probably enough in practice)
+        raise wrap_oserror(space, e, eintr_retry=False)
 
 def ctermid(space):
     """ctermid() -> string
 
     Return the name of the controlling terminal for this process.
     """
-    return space.wrap_fsdecoded(os.ctermid())
+    return space.newfilename(os.ctermid())
 
 @unwrap_spec(fd=c_int)
 def device_encoding(space, fd):
@@ -2084,14 +2191,14 @@
         return space.w_None
     if _WIN32:
         if fd == 0:
-            return space.wrap('cp%d' % rwin32.GetConsoleCP())
+            return space.newtext('cp%d' % rwin32.GetConsoleCP())
         if fd in (1, 2):
-            return space.wrap('cp%d' % rwin32.GetConsoleOutputCP())
+            return space.newtext('cp%d' % rwin32.GetConsoleOutputCP())
     from rpython.rlib import rlocale
     if rlocale.HAVE_LANGINFO:
         codeset = rlocale.nl_langinfo(rlocale.CODESET)
         if codeset:
-            return space.wrap(codeset)
+            return space.newtext(codeset)
     return space.w_None
 
 if _WIN32:
@@ -2102,10 +2209,10 @@
         try:
             info = nt._getfileinformation(fd)
         except OSError as e:
-            raise wrap_oserror(space, e)
-        return space.newtuple([space.wrap(info[0]),
-                               space.wrap(info[1]),
-                               space.wrap(info[2])])
+            raise wrap_oserror(space, e, eintr_retry=False)
+        return space.newtuple([space.newint(info[0]),
+                               space.newint(info[1]),
+                               space.newint(info[2])])
 
     def _getfinalpathname(space, w_path):
         path = space.unicode_w(w_path)
@@ -2113,10 +2220,10 @@
             result = nt._getfinalpathname(path)
         except nt.LLNotImplemented as e:
             raise OperationError(space.w_NotImplementedError,
-                                 space.wrap(e.msg))
+                                 space.newtext(e.msg))
         except OSError as e:
-            raise wrap_oserror2(space, e, w_path)
-        return space.wrap(result)
+            raise wrap_oserror2(space, e, w_path, eintr_retry=False)
+        return space.newunicode(result)
 
 
 def chflags():
@@ -2219,20 +2326,20 @@
             success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
             if not success:
                 raise rwin32.lastSavedWindowsError()
-            w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
-            w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
+            w_columns = space.newint(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
+            w_lines = space.newint(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
     else:
         with lltype.scoped_alloc(rposix.WINSIZE) as winsize:
             failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
             if failed:
                 raise exception_from_saved_errno(space, space.w_OSError)
 
-            w_columns = space.wrap(r_uint(winsize.c_ws_col))
-            w_lines = space.wrap(r_uint(winsize.c_ws_row))
+            w_columns = space.newint(r_uint(winsize.c_ws_col))
+            w_lines = space.newint(r_uint(winsize.c_ws_row))
 
     w_tuple = space.newtuple([w_columns, w_lines])
     w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
-                                    space.wrap('terminal_size'))
+                                    space.newtext('terminal_size'))
 
     return space.call_function(w_terminal_size, w_tuple)
 
@@ -2240,14 +2347,14 @@
     count = rposix.cpu_count()
     if count <= 0:
         return space.w_None
-    return space.wrap(count)
+    return space.newint(count)
 
 @unwrap_spec(fd=c_int)
 def get_blocking(space, fd):
     try:
         flags = rposix.get_status_flags(fd)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
     return space.newbool(flags & rposix.O_NONBLOCK == 0)
 
 @unwrap_spec(fd=c_int, blocking=int)
@@ -2260,4 +2367,4 @@
             flags |= rposix.O_NONBLOCK
         rposix.set_status_flags(fd, flags)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=False)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -7,6 +7,7 @@
 
 from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
 from rpython.translator.c.test.test_extfunc import need_sparse_files
 from rpython.rlib import rposix
 
@@ -201,6 +202,8 @@
         excinfo = raises(TypeError, self.posix.stat, 2.)
         assert "should be string, bytes or integer, not float" in str(excinfo.value)
         raises(ValueError, self.posix.stat, -1)
+        raises(ValueError, self.posix.stat, b"abc\x00def")
+        raises(ValueError, self.posix.stat, u"abc\x00def")
 
     if hasattr(__import__(os.name), "statvfs"):
         def test_statvfs(self):
@@ -359,11 +362,11 @@
         pdir = self.pdir + '/file1'
         posix = self.posix
 
-        assert posix.access(pdir, posix.R_OK)
-        assert posix.access(pdir, posix.W_OK)
+        assert posix.access(pdir, posix.R_OK) is True
+        assert posix.access(pdir, posix.W_OK) is True
         import sys
         if sys.platform != "win32":
-            assert not posix.access(pdir, posix.X_OK)
+            assert posix.access(pdir, posix.X_OK) is False
 
     def test_times(self):
         """
@@ -839,7 +842,7 @@
             fd = os.open(self.path2 + 'test_os_pread', os.O_RDWR | os.O_CREAT)
             try:
                 os.write(fd, b'test')
-                os.lseek(fd, 0, os.SEEK_SET)
+                os.lseek(fd, 0, 0)
                 assert os.pread(fd, 2, 1) == b'es'
                 assert os.read(fd, 2) == b'te'
             finally:
@@ -851,7 +854,7 @@
             fd = os.open(self.path2 + 'test_os_pwrite', os.O_RDWR | os.O_CREAT)
             try:
                 os.write(fd, b'test')
-                os.lseek(fd, 0, os.SEEK_SET)
+                os.lseek(fd, 0, 0)
                 os.pwrite(fd, b'xx', 1)
                 assert os.read(fd, 4) == b'txxt'
             finally:
@@ -874,6 +877,31 @@
         assert st.st_size == 10000000000
     test_largefile.need_sparse_files = True
 
+    if hasattr(rposix, 'getpriority'):
+        def test_os_set_get_priority(self):
+            posix, os = self.posix, self.os
+            childpid = os.fork()
+            if childpid == 0:
+                # in the child (avoids changing the priority of the parent
+                # process)
+                orig_priority = posix.getpriority(posix.PRIO_PROCESS,
+                                                  os.getpid())
+                orig_grp_priority = posix.getpriority(posix.PRIO_PGRP,
+                                                      os.getpgrp())
+                posix.setpriority(posix.PRIO_PROCESS, os.getpid(),
+                                  orig_priority + 1)
+                new_priority = posix.getpriority(posix.PRIO_PROCESS,
+                                                 os.getpid())
+                assert new_priority == orig_priority + 1
+                assert posix.getpriority(posix.PRIO_PGRP, os.getpgrp()) == (
+                    orig_grp_priority)
+                os._exit(0)    # ok
+            #
+            pid1, status1 = os.waitpid(childpid, 0)
+            assert pid1 == childpid
+            assert os.WIFEXITED(status1)
+            assert os.WEXITSTATUS(status1) == 0   # else, test failure
+
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',
@@ -1090,6 +1118,10 @@
             posix.truncate(dest, 1)
             assert 1 == posix.stat(dest).st_size
 
+            # File does not exist
+            e = raises(OSError, posix.truncate, dest + '-DOESNT-EXIST', 0)
+            assert e.value.filename == dest + '-DOESNT-EXIST'
+
     try:
         os.getlogin()
     except (AttributeError, OSError):
@@ -1389,3 +1421,40 @@
         if os.name == 'posix':
             assert os.open in os.supports_dir_fd  # openat()
 
+
+class AppTestPep475Retry:
+    spaceconfig = {'usemodules': USEMODULES}
+
+    def setup_class(cls):
+        if os.name != 'posix':
+            skip("xxx tests are posix-only")
+        if cls.runappdirect:
+            skip("xxx does not work with -A")
+
+        def fd_data_after_delay(space):
+            g = os.popen("sleep 5 && echo hello", "r")
+            cls._keepalive_g = g
+            return space.wrap(g.fileno())
+
+        cls.w_posix = space.appexec([], GET_POSIX)
+        cls.w_fd_data_after_delay = cls.space.wrap(
+            interp2app(fd_data_after_delay))
+
+    def test_pep475_retry_read(self):
+        import _signal as signal
+        signalled = []
+
+        def foo(*args):
+            signalled.append("ALARM")
+
+        signal.signal(signal.SIGALRM, foo)
+        try:
+            fd = self.fd_data_after_delay()
+            signal.alarm(1)
+            got = self.posix.read(fd, 100)
+            self.posix.close(fd)
+        finally:
+            signal.signal(signal.SIGALRM, signal.SIG_DFL)
+
+        assert signalled != []
+        assert got.startswith(b'h')
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -234,6 +234,7 @@
         _ptyh = 'pty.h'
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
+                'sys/resource.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('freebsd'):
@@ -249,6 +250,9 @@
     SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
     SEEK_CUR = rffi_platform.DefinedConstantInteger('SEEK_CUR')
     SEEK_END = rffi_platform.DefinedConstantInteger('SEEK_END')
+    PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
+    PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
+    PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T = rffi_platform.SimpleType('off_t')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
@@ -262,6 +266,7 @@
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
+        ID_T = rffi_platform.SimpleType('id_t', rffi.UINT)
         TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
@@ -1760,6 +1765,22 @@
     def setresgid(rgid, egid, sgid):
         handle_posix_error('setresgid', c_setresgid(rgid, egid, sgid))
 
+    c_getpriority = external('getpriority', [rffi.INT, ID_T], rffi.INT,
+                             save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_setpriority = external('setpriority', [rffi.INT, ID_T, rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    def getpriority(which, who):
+        result = widen(c_getpriority(which, who))
+        error = get_saved_errno()
+        if error != 0:
+            raise OSError(error, 'getpriority failed')
+        return result
+
+    def setpriority(which, who, prio):
+        handle_posix_error('setpriority', c_setpriority(which, who, prio))
+
+
 #___________________________________________________________________
 
 c_chroot = external('chroot', [rffi.CCHARP], rffi.INT,
@@ -1804,25 +1825,23 @@
     finally:
         lltype.free(l_utsbuf, flavor='raw')
 
-# These are actually macros on some/most systems
-c_makedev = external('makedev', [rffi.INT, rffi.INT], rffi.INT)
-c_major = external('major', [rffi.INT], rffi.INT)
-c_minor = external('minor', [rffi.INT], rffi.INT)
+if sys.platform != 'win32':
+    # These are actually macros on some/most systems
+    c_makedev = external('makedev', [rffi.INT, rffi.INT], rffi.INT, macro=True)
+    c_major = external('major', [rffi.INT], rffi.INT, macro=True)
+    c_minor = external('minor', [rffi.INT], rffi.INT, macro=True)
 
- at replace_os_function('makedev')
- at jit.dont_look_inside
-def makedev(maj, min):
-    return c_makedev(maj, min)
+    @replace_os_function('makedev')
+    def makedev(maj, min):
+        return c_makedev(maj, min)
 
- at replace_os_function('major')
- at jit.dont_look_inside
-def major(dev):
-    return c_major(dev)
+    @replace_os_function('major')
+    def major(dev):
+        return c_major(dev)
 
- at replace_os_function('minor')
- at jit.dont_look_inside
-def minor(dev):
-    return c_minor(dev)
+    @replace_os_function('minor')
+    def minor(dev):
+        return c_minor(dev)
 
 #___________________________________________________________________
 

From pypy.commits at gmail.com  Mon Feb 27 06:40:13 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:40:13 -0800 (PST)
Subject: [pypy-commit] pypy pread/pwrite: retry for pwrite also
Message-ID: <58b4101d.97002e0a.4ac70.f329@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: pread/pwrite
Changeset: r90388:891d7b451da6
Date: 2017-02-25 13:30 +0300
http://bitbucket.org/pypy/pypy/changeset/891d7b451da6/

Log:	retry for pwrite also

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -312,7 +312,7 @@
     try:
         res = rposix.pwrite(fd, data.as_str(), offset)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, eintr_retry=True)
     else:
         return space.wrap(res)
 

From pypy.commits at gmail.com  Mon Feb 27 06:40:15 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 03:40:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-text-utf8: (mjacob,
 plan_rich) forbid surrogates while encoding to utf-8 in
 W_UnicodeObject.text_w
Message-ID: <58b4101f.129f190a.1d48b.318a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-text-utf8
Changeset: r90389:08bd2beb79dc
Date: 2017-02-27 12:39 +0100
http://bitbucket.org/pypy/pypy/changeset/08bd2beb79dc/

Log:	(mjacob, plan_rich) forbid surrogates while encoding to utf-8 in
	W_UnicodeObject.text_w

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -8,7 +8,7 @@
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii,
-    unicode_encode_utf8sp)
+    unicode_encode_utf8_forbid_surrogates, SurrogateError)
 from rpython.rlib import jit
 
 from pypy.interpreter import unicodehelper
@@ -82,7 +82,8 @@
 
     def text_w(self, space):
         identifier = jit.conditional_call_elidable(
-                            self._utf8, g_encode_utf8, self._value)
+                            self._utf8, g_encode_utf8, space, self,
+                            self._value)
         if not jit.isconstant(self):
             self._utf8 = identifier
         return identifier
@@ -1255,9 +1256,17 @@
     return u''.join(result)
 
 @jit.elidable
-def g_encode_utf8(value):
+def g_encode_utf8(space, w_value, value):
     """This is a global function because of jit.conditional_call_value"""
-    return unicode_encode_utf8sp(value, len(value))
+    try:
+        return unicode_encode_utf8_forbid_surrogates(value, len(value))
+    except SurrogateError as e:
+        raise OperationError(space.w_UnicodeEncodeError,
+                space.newtuple([space.newtext('utf-8'),
+                                w_value,
+                                space.newint(e.index-1),
+                                space.newint(e.index),
+                                space.newtext("surrogates not allowed")]))
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -433,7 +433,9 @@
     return result.build()
 
 class SurrogateError(Exception):
-    pass
+    def __init__(self, char, index):
+        self.char = char
+        self.index = index
 
 def unicode_encode_utf8_forbid_surrogates(s, size):
     # Strict surrogate-forbidding utf-8 encoding.  Any surrogate character
@@ -454,7 +456,7 @@
             result.append(chr((0x80 | (ch & 0x3f))))
         elif ch < 0x10000:
             if 0xD800 <= ch <= 0xDFFF:
-                raise SurrogateError
+                raise SurrogateError(ch, pos)
             # Encode UCS2 Unicode ordinals
             result.append((chr((0xe0 | (ch >> 12)))))
             result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))

From pypy.commits at gmail.com  Mon Feb 27 06:40:00 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Mon, 27 Feb 2017 03:40:00 -0800 (PST)
Subject: [pypy-commit] pypy get/setpriority: error handling
Message-ID: <58b41010.13542e0a.d079d.0192@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: get/setpriority
Changeset: r90382:e22cb19efefa
Date: 2017-01-17 17:29 +0300
http://bitbucket.org/pypy/pypy/changeset/e22cb19efefa/

Log:	error handling

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1846,10 +1846,11 @@
     
     Get program scheduling priority.
     """
-    try:
-        returned_priority = rposix.c_getpriority(program, identifier)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    error = None
+    returned_priority = rposix.handle_posix_error('getpriority',rposix.c_getpriority(program, identifier))
+    error = rposix.get_saved_errno()
+    if error == 0:
+        raise OSError(error, "getpriority failed")
     return space.wrap(returned_priority)
 
 @unwrap_spec(program=int, identifier=int, priority=int)
@@ -1858,10 +1859,11 @@
     
     Set program scheduling priority.
     """
-    try:
-        rposix.c_setpriority(program, identifier, priority)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    rposix.c_setpriority(program, identifier, priority)
+    error = rposix.get_saved_errno()
+    if error == -1:
+        raise OSError(error, "setpriority failed")
+        
 
 def declare_new_w_star(name):
     if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -11,7 +11,7 @@
 from rpython.rlib import rposix
 
 USEMODULES = ['binascii', 'posix', 'signal', 'struct', 'time']
-# py3k os.open uses subprocess, requiring the following per platform
+# py3k  os.open uses subprocess, requiring the following per platform
 if os.name != 'nt':
     USEMODULES += ['fcntl', 'select', '_posixsubprocess']
 else:

From pypy.commits at gmail.com  Mon Feb 27 06:51:22 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 03:51:22 -0800 (PST)
Subject: [pypy-commit] pypy get/setpriority: close branch,
 accidentally pushed the merge I did for the pull request #517
Message-ID: <58b412ba.0d212e0a.5cd32.34af@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: get/setpriority
Changeset: r90390:76ca536c7972
Date: 2017-02-27 12:50 +0100
http://bitbucket.org/pypy/pypy/changeset/76ca536c7972/

Log:	close branch, accidentally pushed the merge I did for the pull
	request #517


From pypy.commits at gmail.com  Mon Feb 27 07:04:37 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 27 Feb 2017 04:04:37 -0800 (PST)
Subject: [pypy-commit] pyrepl default: manually apply pypy commit
 ca9508369e5a, to keep compatibility between the twos
Message-ID: <58b415d5.951c190a.1160e.3604@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r264:62630c596d7c
Date: 2017-02-27 13:04 +0100
http://bitbucket.org/pypy/pyrepl/changeset/62630c596d7c/

Log:	manually apply pypy commit ca9508369e5a, to keep compatibility
	between the twos

diff --git a/pyrepl/simple_interact.py b/pyrepl/simple_interact.py
--- a/pyrepl/simple_interact.py
+++ b/pyrepl/simple_interact.py
@@ -35,11 +35,13 @@
     return True
 
 
-def run_multiline_interactive_console(mainmodule=None):
+def run_multiline_interactive_console(mainmodule=None, future_flags=0):
     import code
     import __main__
     mainmodule = mainmodule or __main__
     console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
 
     def more_lines(unicodetext):
         # ooh, look at the hack:

From pypy.commits at gmail.com  Mon Feb 27 08:10:38 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 27 Feb 2017 05:10:38 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: Add W_UnicodeObject._multi_chr()
 returning str
Message-ID: <58b4254e.5ed7190a.805f6.3c2c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-utf8
Changeset: r90391:df262c600edd
Date: 2017-02-27 14:09 +0100
http://bitbucket.org/pypy/pypy/changeset/df262c600edd/

Log:	Add W_UnicodeObject._multi_chr() returning str

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -158,6 +158,9 @@
         assert len(char) == 1
         return char[0]
 
+    def _multi_chr(self, unichar):
+        return unichar.encode('utf8')
+
     _builder = UnicodeBuilder
 
     def _isupper(self, ch):

From pypy.commits at gmail.com  Mon Feb 27 08:26:56 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 05:26:56 -0800 (PST)
Subject: [pypy-commit] pypy default: add use autodoc for text_w
Message-ID: <58b42920.8e48190a.65b34.2e1b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r90392:610a99f1638f
Date: 2017-02-27 14:03 +0100
http://bitbucket.org/pypy/pypy/changeset/610a99f1638f/

Log:	add use autodoc for text_w

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -17,6 +17,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.append(os.path.abspath('.'))
+sys.path.append(os.path.abspath('../../'))
 
 
 # -- Read The Docs theme config ------------------------------------------------
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -291,10 +291,7 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
-.. py:function:: text_w(w_x)
-
-   Takes an application level :py:class:`str` and converts it to a rpython byte string.
-   PyPy3 this method will return an utf-8-nosg encoded result.
+.. automethod:: pypy.interpreter.baseobjspace.ObjSpace.text_w(w_x)
 
 .. py:function:: bytes_w(w_x)
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1613,8 +1613,14 @@
         return w_obj.str_w(self)
 
     def text_w(self, w_obj):
-        """Takes a string object (unicode in Python 3) and returns an
-        unwrapped RPython bytestring."""
+        """ PyPy2 takes either a :py:class:`str` and returns a
+            rpython byte string, or it takes an :py:class:`unicode`
+            and uses the systems default encoding to return a rpython
+            byte string.
+
+            On PyPy3 it takes a :py:class:`str` and it will return
+            an utf-8 encoded rpython string.
+        """
         return w_obj.str_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w

From pypy.commits at gmail.com  Mon Feb 27 08:34:13 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 27 Feb 2017 05:34:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: (plan_rich, mjacob) hg merge default
Message-ID: <58b42ad5.4c142e0a.d17e0.e4c1@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r90393:2e9fdb0ad351
Date: 2017-02-27 14:34 +0100
http://bitbucket.org/pypy/pypy/changeset/2e9fdb0ad351/

Log:	(plan_rich, mjacob) hg merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -60,6 +60,9 @@
 ^lib_pypy/ctypes_config_cache/_.+_cache\.py$
 ^lib_pypy/ctypes_config_cache/_.+_.+_\.py$
 ^lib_pypy/_libmpdec/.+.o$
+^lib_pypy/.+.c$
+^lib_pypy/.+.o$
+^lib_pypy/.+.so$
 ^pypy/doc/discussion/.+\.html$
 ^include/.+\.h$
 ^include/.+\.inl$
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -49,7 +49,7 @@
 -------------------------------
 (**Note**: for some hints on how to translate the Python interpreter under
 Windows, see the `windows document`_ . For hints on how to cross-compile in
-a chroot using scratchbox2, see the `arm document`_ in the 
+a chroot using scratchbox2, see the `arm document`_ in the
 `RPython documentation`_)
 
 .. _`windows document`: windows.html
@@ -57,7 +57,7 @@
 .. _`RPython documentation`: http://rpython.readthedocs.org
 
 The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
-already installed. If translating on CPython, you need to install it, e.g. 
+already installed. If translating on CPython, you need to install it, e.g.
 using ``pip install cffi``.
 
 To build PyPy on Unix using the C translation backend, you need at least a C
@@ -113,7 +113,7 @@
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
-    lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
+    sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
     gdbm-devel \
     xz-devel  # For lzma on PyPy3.
 
@@ -183,7 +183,7 @@
 imported the first time.
 
 ::
-    
+
     cd pypy/tool/release
     ./package.py pypy-VER-PLATFORM
 
@@ -220,5 +220,3 @@
 to continue normally. If the default path is usable, most code will be fine.
 However, the ``sys.prefix`` will be unset and some existing libraries assume
 that this is never the case.
-
-
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -17,6 +17,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.append(os.path.abspath('.'))
+sys.path.append(os.path.abspath('../../'))
 
 
 # -- Read The Docs theme config ------------------------------------------------
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -437,3 +437,11 @@
 but so far there doesn't seem to be an overwhelming commercial interest in it.
 
 .. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation
+
+
+How long will PyPy support Python2?
+-----------------------------------
+
+Since RPython is built on top of Python2 and that is extremely unlikely to
+change, the Python2 version of PyPy will be around "forever", i.e. as long as
+PyPy itself is around.
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -291,10 +291,7 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
-.. py:function:: text_w(w_x)
-
-   Takes an application level :py:class:`str` and converts it to a rpython byte string.
-   PyPy3 this method will return an utf-8-nosg encoded result.
+.. automethod:: pypy.interpreter.baseobjspace.ObjSpace.text_w(w_x)
 
 .. py:function:: bytes_w(w_x)
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1580,6 +1580,21 @@
     def text_or_none_w(self, w_obj):
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
+    def bytes_w(self, w_obj):
+        "Takes a bytes object and returns an unwrapped RPython bytestring."
+        return w_obj.bytes_w(self)
+
+    def text_w(self, w_obj):
+        """ PyPy2 takes either a :py:class:`str` and returns a
+            rpython byte string, or it takes an :py:class:`unicode`
+            and uses the systems default encoding to return a rpython
+            byte string.
+
+            On PyPy3 it takes a :py:class:`str` and it will return
+            an utf-8 encoded rpython string.
+        """
+        return w_obj.text_w(self)
+
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
     def str_w(self, w_obj):
         """
@@ -1594,9 +1609,6 @@
         else:
             return w_obj.bytes_w(self)
 
-    def bytes_w(self, w_obj):
-        return w_obj.bytes_w(self)
-
     def bytes0_w(self, w_obj):
         "Like bytes_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
@@ -1665,14 +1677,6 @@
                         "characters")
         return rstring.assert_str0(result)
 
-    def text_w(self, w_obj):
-        """
-        Unwrap a unicode object and return a 'utf-8-nosg' byte string
-        ('no surrogate').  This encoding always works and is in one-to-
-        one correspondance with the unicode.
-        """
-        return w_obj.text_w(self)
-
     realtext_w = text_w         # Python 2 compatibility
     realunicode_w = unicode_w
 

From pypy.commits at gmail.com  Mon Feb 27 08:41:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 27 Feb 2017 05:41:51 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: waaaaaaaaa but revdb is cool
Message-ID: <58b42c9f.6718190a.d623e.1c60@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: unicode-utf8
Changeset: r90394:e0ed90dd707f
Date: 2017-02-27 14:40 +0100
http://bitbucket.org/pypy/pypy/changeset/e0ed90dd707f/

Log:	waaaaaaaaa but revdb is cool

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1968,7 +1968,7 @@
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
-    _none_value = None
+    _none_value = ""
 
     def wrap(self, stringval):
         return self.space.newbytes(stringval)
@@ -2000,7 +2000,7 @@
 class UnicodeListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
-    _none_value = None
+    _none_value = u""
 
     def wrap(self, stringval):
         assert stringval is not None

From pypy.commits at gmail.com  Mon Feb 27 08:41:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 27 Feb 2017 05:41:53 -0800 (PST)
Subject: [pypy-commit] pypy default: cherry-pick e0ed90dd707f
Message-ID: <58b42ca1.43502e0a.b5243.e076@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90395:ce15e13c6312
Date: 2017-02-27 14:40 +0100
http://bitbucket.org/pypy/pypy/changeset/ce15e13c6312/

Log:	cherry-pick e0ed90dd707f

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1966,7 +1966,7 @@
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
-    _none_value = None
+    _none_value = ""
 
     def wrap(self, stringval):
         return self.space.newbytes(stringval)
@@ -1998,7 +1998,7 @@
 class UnicodeListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
-    _none_value = None
+    _none_value = u""
 
     def wrap(self, stringval):
         assert stringval is not None

From pypy.commits at gmail.com  Mon Feb 27 08:41:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 27 Feb 2017 05:41:55 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: merge heads
Message-ID: <58b42ca3.46542e0a.7b5cd.e2b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: unicode-utf8
Changeset: r90396:2fa0248d61eb
Date: 2017-02-27 14:41 +0100
http://bitbucket.org/pypy/pypy/changeset/2fa0248d61eb/

Log:	merge heads

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -158,6 +158,9 @@
         assert len(char) == 1
         return char[0]
 
+    def _multi_chr(self, unichar):
+        return unichar.encode('utf8')
+
     _builder = UnicodeBuilder
 
     def _isupper(self, ch):

From pypy.commits at gmail.com  Mon Feb 27 08:42:50 2017
From: pypy.commits at gmail.com (fijal)
Date: Mon, 27 Feb 2017 05:42:50 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: remove pdb
Message-ID: <58b42cda.07582e0a.99b4a.dbec@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90397:f5e948aa5162
Date: 2017-02-27 14:42 +0100
http://bitbucket.org/pypy/pypy/changeset/f5e948aa5162/

Log:	remove pdb

diff --git a/rpython/annotator/listdef.py b/rpython/annotator/listdef.py
--- a/rpython/annotator/listdef.py
+++ b/rpython/annotator/listdef.py
@@ -107,9 +107,6 @@
             self.bookkeeper.annotator.reflowfromposition(position_key)
 
     def generalize(self, s_other_value):
-        if hasattr(self.s_value, 'can_be_None') and not self.s_value.can_be_None and getattr(s_other_value, 'can_be_None', False):
-            import pdb
-            pdb.set_trace()
         s_new_value = unionof(self.s_value, s_other_value)
         updated = s_new_value != self.s_value
         if updated:

From pypy.commits at gmail.com  Mon Feb 27 08:59:24 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 05:59:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: (mjacob, plan_rich) merge default
Message-ID: <58b430bc.0d50190a.fe37e.ba3f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90399:6dd6cb21c7c9
Date: 2017-02-27 14:58 +0100
http://bitbucket.org/pypy/pypy/changeset/6dd6cb21c7c9/

Log:	(mjacob, plan_rich) merge default

diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -291,13 +291,9 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
+.. automethod:: pypy.interpreter.baseobjspace.ObjSpace.bytes_w(w_x)
 .. automethod:: pypy.interpreter.baseobjspace.ObjSpace.text_w(w_x)
 
-.. py:function:: bytes_w(w_x)
-
-   Takes an application level :py:class:`bytes` (PyPy2 this equals `str`) and returns a rpython
-   byte string.
-
 .. py:function:: str_w(w_x)
 
    **Deprecated. use text_w or bytes_w instead**
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1581,7 +1581,9 @@
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
     def bytes_w(self, w_obj):
-        "Takes a bytes object and returns an unwrapped RPython bytestring."
+        """ Takes an application level :py:class:`bytes`
+            (on PyPy2 this equals `str`) and returns a rpython byte string.
+        """
         return w_obj.bytes_w(self)
 
     def text_w(self, w_obj):
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1945,7 +1945,7 @@
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
-    _none_value = None
+    _none_value = ""
 
     def wrap(self, stringval):
         return self.space.newbytes(stringval)
@@ -1977,7 +1977,7 @@
 class UnicodeListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
 
-    _none_value = None
+    _none_value = u""
 
     def wrap(self, stringval):
         assert stringval is not None

From pypy.commits at gmail.com  Mon Feb 27 08:59:22 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 05:59:22 -0800 (PST)
Subject: [pypy-commit] pypy default: (mjacob,
 plan_rich) update bytes_w to autodoc
Message-ID: <58b430ba.1e142e0a.57aef.314a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r90398:14bcde218123
Date: 2017-02-27 14:57 +0100
http://bitbucket.org/pypy/pypy/changeset/14bcde218123/

Log:	(mjacob, plan_rich) update bytes_w to autodoc

diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -291,13 +291,9 @@
    If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level
    :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`.
 
+.. automethod:: pypy.interpreter.baseobjspace.ObjSpace.bytes_w(w_x)
 .. automethod:: pypy.interpreter.baseobjspace.ObjSpace.text_w(w_x)
 
-.. py:function:: bytes_w(w_x)
-
-   Takes an application level :py:class:`bytes` (PyPy2 this equals `str`) and returns a rpython
-   byte string.
-
 .. py:function:: str_w(w_x)
 
    **Deprecated. use text_w or bytes_w instead**
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1609,7 +1609,9 @@
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
     def bytes_w(self, w_obj):
-        "Takes a bytes object and returns an unwrapped RPython bytestring."
+        """ Takes an application level :py:class:`bytes`
+            (on PyPy2 this equals `str`) and returns a rpython byte string.
+        """
         return w_obj.str_w(self)
 
     def text_w(self, w_obj):

From pypy.commits at gmail.com  Mon Feb 27 09:02:58 2017
From: pypy.commits at gmail.com (fijal)
Date: Mon, 27 Feb 2017 06:02:58 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: fixes
Message-ID: <58b43192.94472e0a.51268.a22f@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90400:17031d8a78ec
Date: 2017-02-27 15:02 +0100
http://bitbucket.org/pypy/pypy/changeset/17031d8a78ec/

Log:	fixes

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -57,7 +57,6 @@
             assert 0 <= ps <= q
             substr = s[ps:q]
         else:
-            xxx
             substr = decode_unicode_utf8(space, s, ps, q)
         if rawmode:
             v, length = unicodehelper.decode_raw_unicode_escape(space, substr)
@@ -72,7 +71,8 @@
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
-            w_u = space.newunicode(unicodehelper.decode_utf8(space, substr))
+            utf, lgt = unicodehelper.decode_utf8(space, substr)
+            w_u = space.newutf8(utf, lgt)
             w_v = unicodehelper.encode(space, w_u, encoding)
             return w_v
         else:
@@ -222,8 +222,8 @@
     # while (s < end && *s != '\\') s++; */ /* inefficient for u".."
     while ps < end and ord(s[ps]) & 0x80:
         ps += 1
-    u = unicodehelper.decode_utf8(space, s[pt:ps])
-    return u, ps
+    utf, _ = unicodehelper.decode_utf8(space, s[pt:ps])
+    return utf.decode('utf8'), ps
 
 def decode_utf8_recode(space, s, ps, end, recode_encoding):
     u, ps = decode_utf8(space, s, ps, end)
diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -50,7 +50,7 @@
         s = "u'\x81'"
         s = s.decode("koi8-u").encode("utf8")
         w_ret = parsestring.parsestr(self.space, 'koi8-u', s)
-        ret = space.unwrap(w_ret)
+        ret = w_ret._utf8.decode('utf8')
         assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'")
 
     def test_unicode_literals(self):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -77,6 +77,13 @@
         errorhandler=raise_unicode_exception_encode,
         allow_surrogates=True)
 
+def decode_utf8(space, s):
+    u, _ = runicode.str_decode_utf_8(s, len(s),
+        "strict", final=True,
+        errorhandler=decode_error_handler(space),
+        allow_surrogates=True)
+    return u.encode('utf8'), len(u)
+
 def utf8_encode_ascii(utf8, utf8len, errors, errorhandler):
     if len(utf8) == utf8len:
         return utf8
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -2,7 +2,7 @@
 
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin,
-    enforceargs, newlist_hint)
+    enforceargs, newlist_hint, specialize)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder, split, rsplit, UnicodeBuilder
 from rpython.rlib.runicode import make_unicode_escape_function
@@ -116,9 +116,8 @@
         return W_UnicodeObject(value.encode('utf8'), len(value))
 
     def _new_from_list(self, value):
-        xxx
-        return W_UnicodeObject(u''.join(value))
-
+        u = u''.join(value)
+        return W_UnicodeObject(u.encode('utf8'), len(u))
     def _empty(self):
         return W_UnicodeObject.EMPTY
 
@@ -154,12 +153,13 @@
     def convert_to_w_unicode(self, space):
         return self
 
+    @specialize.argtype(1)
     def _chr(self, char):
         assert len(char) == 1
         return char[0]
 
     def _multi_chr(self, unichar):
-        return unichar.encode('utf8')
+        return unichar
 
     _builder = UnicodeBuilder
 
@@ -387,7 +387,7 @@
     def descr_join(self, space, w_list):
         l = space.listview_unicode(w_list)
         if l is not None:
-            xxx
+            assert False, "unreachable"
             if len(l) == 1:
                 return space.newunicode(l[0])
             return space.newunicode(self._utf8).join(l)
@@ -513,7 +513,7 @@
     def descr_zfill(self, space, width):
         selfval = self._utf8
         if len(selfval) == 0:
-            return W_UnicodeObject(self._multi_chr(self._chr('0')) * width, width)
+            return W_UnicodeObject(self._chr('0') * width, width)
         num_zeros = width - self._len()
         if num_zeros <= 0:
             # cannot return self, in case it is a subclass of str
@@ -571,7 +571,7 @@
         d = width - self._len()
         if d > 0:
             offset = d//2 + (d & width & 1)
-            fillchar = self._multi_chr(fillchar[0])
+            fillchar = fillchar[0]
             centered = offset * fillchar + value + (d - offset) * fillchar
         else:
             centered = value

From pypy.commits at gmail.com  Mon Feb 27 09:12:17 2017
From: pypy.commits at gmail.com (fijal)
Date: Mon, 27 Feb 2017 06:12:17 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: fix
Message-ID: <58b433c1.43502e0a.b5243.e210@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90401:e4861071cd46
Date: 2017-02-27 15:11 +0100
http://bitbucket.org/pypy/pypy/changeset/e4861071cd46/

Log:	fix

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -42,7 +42,7 @@
     # XXX that guy does not belong in runicode (nor in rutf8)
     result_u, consumed = runicode.str_decode_unicode_escape(
         string, len(string), "strict",
-        final=True, errorhandler=decode_error_handler(space),
+        final=True, errorhandler=DecodeWrapper(decode_error_handler(space)).handle,
         unicodedata_handler=unicodedata_handler)
     return result_u.encode('utf8'), len(result_u)
 
@@ -51,7 +51,7 @@
     # XXX that guy does not belong in runicode (nor in rutf8)
     result_u, consumed = runicode.str_decode_raw_unicode_escape(
         string, len(string), "strict",
-        final=True, errorhandler=decode_error_handler(space))
+        final=True, errorhandler=DecodeWrapper(decode_error_handler(space)).handle)
     return result_u.encode('utf8'), len(result_u)
 
 def check_utf8(space, string):

From pypy.commits at gmail.com  Mon Feb 27 09:27:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 27 Feb 2017 06:27:27 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: Add a test
Message-ID: <58b4374f.14db190a.78260.38a7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: unicode-utf8
Changeset: r90402:0a067e948514
Date: 2017-02-27 15:26 +0100
http://bitbucket.org/pypy/pypy/changeset/0a067e948514/

Log:	Add a test

diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -146,6 +146,7 @@
         raises(ValueError, u'abc'.split, u'')
         raises(ValueError, 'abc'.split, u'')
         assert u'   a b c d'.split(None, 0) == [u'a b c d']
+        assert u'a\nb\u1680c'.split() == [u'a', u'b', u'c']
 
     def test_rsplit(self):
         assert u"".rsplit() == []

From pypy.commits at gmail.com  Mon Feb 27 09:38:31 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 06:38:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-text-utf8: (ronan, arigato,
 plan_rich) make the W_UnicodeObject elidable by the jit,
 the exception is propagated one level up
Message-ID: <58b439e7.97002e0a.4ac70.fc11@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-text-utf8
Changeset: r90403:2694e2b25754
Date: 2017-02-27 15:37 +0100
http://bitbucket.org/pypy/pypy/changeset/2694e2b25754/

Log:	(ronan, arigato, plan_rich) make the W_UnicodeObject elidable by the
	jit, the exception is propagated one level up

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -81,9 +81,16 @@
         return self._value
 
     def text_w(self, space):
-        identifier = jit.conditional_call_elidable(
-                            self._utf8, g_encode_utf8, space, self,
-                            self._value)
+        try:
+            identifier = jit.conditional_call_elidable(
+                                self._utf8, g_encode_utf8, self._value)
+        except SurrogateError as e:
+            raise OperationError(space.w_UnicodeEncodeError,
+                    space.newtuple([space.newtext('utf-8'),
+                                    self,
+                                    space.newint(e.index-1),
+                                    space.newint(e.index),
+                                    space.newtext("surrogates not allowed")]))
         if not jit.isconstant(self):
             self._utf8 = identifier
         return identifier
@@ -1256,17 +1263,9 @@
     return u''.join(result)
 
 @jit.elidable
-def g_encode_utf8(space, w_value, value):
+def g_encode_utf8(value):
     """This is a global function because of jit.conditional_call_value"""
-    try:
-        return unicode_encode_utf8_forbid_surrogates(value, len(value))
-    except SurrogateError as e:
-        raise OperationError(space.w_UnicodeEncodeError,
-                space.newtuple([space.newtext('utf-8'),
-                                w_value,
-                                space.newint(e.index-1),
-                                space.newint(e.index),
-                                space.newtext("surrogates not allowed")]))
+    return unicode_encode_utf8_forbid_surrogates(value, len(value))
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')

From pypy.commits at gmail.com  Mon Feb 27 10:23:24 2017
From: pypy.commits at gmail.com (fijal)
Date: Mon, 27 Feb 2017 07:23:24 -0800 (PST)
Subject: [pypy-commit] pypy unicode-utf8: fix and a workaround
Message-ID: <58b4446c.83c3190a.36b0d.d8e1@mx.google.com>

Author: fijal
Branch: unicode-utf8
Changeset: r90404:737c72b15c6d
Date: 2017-02-27 16:22 +0100
http://bitbucket.org/pypy/pypy/changeset/737c72b15c6d/

Log:	fix and a workaround

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -303,6 +303,13 @@
         return self.newlist(list_u)
         return W_ListObject.newlist_unicode(self, list_u)
 
+    def newlist_from_unicode(self, lst):
+        res_w = []
+        for u in lst:
+            assert u is not None
+            res_w.append(self.newutf8(u, -1))
+        return self.newlist(res_w)
+
     def newlist_int(self, list_i):
         return W_ListObject.newlist_int(self, list_i)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -109,7 +109,7 @@
         if self._len() != 1:
             raise oefmt(space.w_TypeError,
                          "ord() expected a character, but string of length %d "
-                         "found", len(self._value))
+                         "found", self._len())
         return space.newint(rutf8.codepoint_at_pos(self._utf8, 0))
 
     def _new(self, value):
@@ -126,6 +126,9 @@
             self._length = self._compute_length()
         return self._length
 
+    def _compute_length(self):
+        return rutf8.compute_length_utf8(self._utf8)
+
     def _val(self, space):
         return self._utf8.decode('utf8')
 
@@ -156,7 +159,7 @@
     @specialize.argtype(1)
     def _chr(self, char):
         assert len(char) == 1
-        return char[0]
+        return unichr(ord(char[0]))
 
     def _multi_chr(self, unichar):
         return unichar
@@ -513,7 +516,7 @@
     def descr_zfill(self, space, width):
         selfval = self._utf8
         if len(selfval) == 0:
-            return W_UnicodeObject(self._chr('0') * width, width)
+            return W_UnicodeObject('0' * width, width)
         num_zeros = width - self._len()
         if num_zeros <= 0:
             # cannot return self, in case it is a subclass of str
@@ -525,7 +528,7 @@
             start = 1
         else:
             start = 0
-        builder.append_multiple_char(self._chr('0'), num_zeros)
+        builder.append_multiple_char('0', num_zeros)
         builder.append_slice(selfval, start, len(selfval))
         return W_UnicodeObject(builder.build(), width)
 
@@ -536,14 +539,14 @@
         value = self._utf8
         if space.is_none(w_sep):
             res = split(value, maxsplit=maxsplit)
-            return space.newlist([W_UnicodeObject(s, -1) for s in res])
+            return space.newlist_from_unicode(res)
 
         by = self.convert_arg_to_w_unicode(space, w_sep)._utf8
         if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = split(value, by, maxsplit)
 
-        return space.newlist([W_UnicodeObject(s, -1) for s in res])
+        return space.newlist_from_unicode(res)
 
     @unwrap_spec(maxsplit=int)
     def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
@@ -551,14 +554,14 @@
         value = self._utf8
         if space.is_none(w_sep):
             res = rsplit(value, maxsplit=maxsplit)
-            return space.newlist([W_UnicodeObject(s, -1) for s in res])
+            return space.newlist_from_unicode(res)
 
         by = self.convert_arg_to_w_unicode(space, w_sep)._utf8
         if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = rsplit(value, by, maxsplit)
 
-        return space.newlist([W_UnicodeObject(s, -1) for s in res])
+        return space.newlist_from_unicode(res)
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_center(self, space, width, w_fillchar):
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -66,6 +66,14 @@
         return pos + 1
     return pos + ord(runicode._utf8_code_length[chr1 - 0x80])
 
+def compute_length_utf8(s):
+    pos = 0
+    lgt = 0
+    while pos < len(s):
+        pos = next_codepoint_pos(s, pos)
+        lgt += 1
+    return lgt
+
 def codepoint_at_pos(code, pos):
     """ Give a codepoint in code at pos - assumes valid utf8, no checking!
     """
diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py
--- a/rpython/rtyper/rmodel.py
+++ b/rpython/rtyper/rmodel.py
@@ -359,6 +359,10 @@
     def ll_str(self, nothing): raise AssertionError("unreachable code")
 impossible_repr = VoidRepr()
 
+class __extend__(pairtype(Repr, VoidRepr)):
+    def convert_from_to((r_from, r_to), v, llops):
+        return inputconst(lltype.Void, None)
+
 class SimplePointerRepr(Repr):
     "Convenience Repr for simple ll pointer types with no operation on them."
 

From pypy.commits at gmail.com  Mon Feb 27 10:32:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 27 Feb 2017 07:32:22 -0800 (PST)
Subject: [pypy-commit] pypy default: Test and fix (from unicode-utf8): an
 obscure case where we get
Message-ID: <58b44686.4b1b190a.d2572.396e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90405:98f4a2ce0fa5
Date: 2017-02-27 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/98f4a2ce0fa5/

Log:	Test and fix (from unicode-utf8): an obscure case where we get
	convert_from_to() that targets SomeImpossibleValue.

diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py
--- a/rpython/rtyper/rmodel.py
+++ b/rpython/rtyper/rmodel.py
@@ -359,6 +359,10 @@
     def ll_str(self, nothing): raise AssertionError("unreachable code")
 impossible_repr = VoidRepr()
 
+class __extend__(pairtype(Repr, VoidRepr)):
+    def convert_from_to((r_from, r_to), v, llops):
+        return inputconst(lltype.Void, None)
+
 class SimplePointerRepr(Repr):
     "Convenience Repr for simple ll pointer types with no operation on them."
 
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -1746,6 +1746,29 @@
         res = self.interpret(g, [1])
         assert res == True
 
+    def test_convert_from_anything_to_impossible(self):
+        def f1():
+            return 42
+        def f2():
+            raise ValueError
+        def f3():
+            raise ValueError
+        def f(i):
+            if i > 5:
+                f = f2
+            else:
+                f = f3
+            try:
+                f()
+            except ValueError:
+                pass
+            if i > 1:
+                f = f2
+            else:
+                f = f1
+            return f()
+        self.interpret(f, [-5])
+
 # ____________________________________________________________
 
 def test_hlinvoke_simple():

From pypy.commits at gmail.com  Mon Feb 27 10:44:13 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 07:44:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-text-utf8: (ronan,
 plan_rich) allow surrogates to be passed to __import__
Message-ID: <58b4494d.039d190a.a2688.208c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-text-utf8
Changeset: r90406:cb8bf6a7d22d
Date: 2017-02-27 16:42 +0100
http://bitbucket.org/pypy/pypy/changeset/cb8bf6a7d22d/

Log:	(ronan, plan_rich) allow surrogates to be passed to __import__

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -67,7 +67,7 @@
 lib_pypy = os.path.join(os.path.dirname(__file__),
                         '..', '..', '..', 'lib_pypy')
 
- at unwrap_spec(modulename='text0', level=int)
+ at unwrap_spec(modulename='fsencode', level=int)
 def importhook(space, modulename, w_globals=None, w_locals=None, w_fromlist=None, level=0):
     # A minimal version, that can only import builtin and lib_pypy modules!
     assert w_locals is w_globals
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -5,6 +5,7 @@
 from pypy.interpreter.pycode import PyCode
 from pypy.module._io.interp_iobase import W_IOBase
 from pypy.interpreter.streamutil import wrap_streamerror
+from pypy.interpreter.error import OperationError
 
 
 def extension_suffixes(space):
@@ -72,7 +73,11 @@
     return None
 
 def is_builtin(space, w_name):
-    name = space.text0_w(w_name)
+    try:
+        name = space.text0_w(w_name)
+    except OperationError:
+        return space.newint(0)
+
     if name not in space.builtin_modules:
         return space.newint(0)
     if space.finditem(space.sys.get('modules'), w_name) is not None:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -255,6 +255,10 @@
     def test_import_keywords(self):
         __import__(name='sys', level=0)
 
+    def test_import_nonutf8_encodable(self):
+        exc = raises(ImportError, __import__, '\ud800')
+        assert exc.value.args[0].startswith("No module named ")
+
     def test_import_by_filename(self):
         import pkg.a
         filename = pkg.a.__file__

From pypy.commits at gmail.com  Mon Feb 27 10:58:26 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 27 Feb 2017 07:58:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-text-utf8: Adapt test for changed behavior.
Message-ID: <58b44ca2.12ce190a.e58af.4004@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5-text-utf8
Changeset: r90407:ae315621f4b7
Date: 2017-02-27 16:59 +0100
http://bitbucket.org/pypy/pypy/changeset/ae315621f4b7/

Log:	Adapt test for changed behavior.

diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -35,8 +35,7 @@
         w_uni = space.wrap(u'abcd')
         assert space.text_w(w_uni) == 'abcd'
         w_uni = space.wrap(unichr(0xd921) + unichr(0xdddd))
-        assert space.text_w(w_uni) == '\xed\xa4\xa1\xed\xb7\x9d'
-        #                             ^^^ and not the 4-bytes combined character
+        raises(UnicodeEncodeError, space.text_w, w_uni)
 
 
 class AppTestUnicodeStringStdOnly:

From pypy.commits at gmail.com  Mon Feb 27 11:40:10 2017
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 27 Feb 2017 08:40:10 -0800 (PST)
Subject: [pypy-commit] stmgc c8-reshare-pages: merge default
Message-ID: <58b4566a.cd092e0a.85d09.1285@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: c8-reshare-pages
Changeset: r2016:afef19229966
Date: 2017-02-27 17:00 +0100
http://bitbucket.org/pypy/stmgc/changeset/afef19229966/

Log:	merge default

diff too long, truncating to 2000 out of 3600 lines

diff --git a/README.txt b/README.md
rename from README.txt
rename to README.md
--- a/README.txt
+++ b/README.md
@@ -1,28 +1,250 @@
 
-STM-GC
-======
+# STM-GC
 
 Welcome!
 
 This is a C library that combines a GC with STM capabilities.
 It is meant to be a general library that can be used in C programs.
 
-The library interface is in "c4/stmgc.h".
+The library interface is in `c8/stmgc.h`.
 
 Progress (these revisions are roughly stable versions, pick the last one):
-- 3aea86a96daf: last rev of "c3", the previous version
-- f1ccf5bbcb6f: first step, working but with no GC
-- 8da924453f29: minor collection seems to be working, no major GC
-- e7249873dcda: major collection seems to be working
 
-The file "c4/doc-objects.txt" contains some low-level explanations.
+ - 3af462f
 
-Run tests with "py.test".
+Run tests with `py.test`.
 
-A demo program can be found in "c4/demo1.c".
-It can be built with "make debug-demo1" or "make build-demo1".
+Demo programs can be found in `c8/demo/`.
 
 The plan is to use this C code directly with PyPy, and not write
 manually the many calls to the shadow stack and the barrier functions.
 But the manual way is possible too, say when writing a small interpreter
 directly in C.
+
+
+# Other resources
+
+http://doc.pypy.org/en/latest/stm.html
+
+# How to run things
+
+## Get PyPy 
+
+ 1. `hg clone https://bitbucket.org/pypy/pypy` (this will take a while, but you
+    can continue with the instructions below)
+ 2. `hg checkout stmgc-c8`
+
+
+## Get STMGC
+
+ 1. `hg clone https://bitbucket.org/pypy/stmgc`
+ 2. `gcc-seg-gs/README.txt` mentions which GCC version should work. Maybe -fXXX
+    flags mentioned at the end are still needed for compiling full PyPy-STM
+
+The folder `c8` contains the current version of the STMGC library
+
+### Project layout of c8
+
+ - `stmgc.h`: the main header file for the library
+ - `stm/`: 
+ 
+    For the GC part: 
+    
+     - `nursery`: minor collection
+     - `gcpage`: major collection
+     - `largemalloc`, `smallmalloc`: object allocation
+     - `finalizer`: object finalizer support
+     - `weakref`: weak references support
+     
+    For the STM part:
+    
+     - `core`: commit, abort, barrier logic of STM
+     - `sync`: segment management and thread support
+     - `pages`: management of page metadata
+     - `signal_handler`: manages pages together with `pages`
+     - `locks`: a set of locks to protect segments
+     - `rewind_setjmp`: setjmp/longjmp implementation that supports arbitrary rollback
+     - `forksupport`: support for forking an STM process
+     - `extra`: on-commit and on-abort callback mechanism
+     - `detach`: transaction detach mechanism (optimised transactional zones)
+     - `setup`: sets up the memory layout and segments
+     
+    Misc:
+    
+     - `fprintcolor`: colourful debug output
+     - `hash_id`: PyPy-compatible identity and identity-hash functionality
+     - `hashtable`: transactional hash table implementation
+     - `queue`: transactional work-queue implementation
+     - `list`: simple growable list implementation
+     - `marker`, `prof`: mechanism to record events
+     - `misc`: mostly debug and testing interface
+     - `pagecopy`: fast copy implementation for pages
+     - `prebuilt`: logic for PyPy's prebuilt objects
+       
+
+
+### Running Tests
+
+Tests are written in Python that calls the C-library through CFFI (Python package).
+
+ 1. install `pytest` and `cffi` packages for Python (via `pip`)
+ 2. running `py.test` in `c8/test` should run all the tests (alternatively, the
+    PyPy-checkout has a pytest.py script in its root-folder, which should work
+    too)
+
+### Running Demos
+
+Demos are small C programs that use the STMGC library directly. They sometimes
+expose real data-races that the sequential Python tests cannot expose.
+
+ 1. for 
+ example: `make build-demo_random2`
+ 2. then run `./build-demo_random2`
+ 
+ 
+### Debugging
+
+GDB works fine for debugging programs with the STMGC library. However, you have
+to tell GDB to ignore `SIGSEGV` by default. A `.gdbinit` could look like this:
+
+    handle SIGSEGV nostop pass noprint
+
+    define sigon
+        handle SIGSEGV stop nopass print
+    end
+
+    define sigoff
+        handle SIGSEGV nostop pass noprint
+    end
+
+    define lon
+        set scheduler-locking on
+    end
+    define loff
+        set scheduler-locking off
+    end
+    
+    # run until crash
+    define runloop
+        set pagination off
+        p $_exitcode = 0
+        while $_exitcode == 0
+            p $_exitcode = -1
+            r
+        end
+        set pagination on
+    end
+
+
+The commands `sigon` and `sigoff` enable and disable `SIGSEGV`-handling. `lon`
+and `loff` enables and disables stopping of other threads while stepping through
+one of them. After reaching a breakpoint in GDB, I usually run `sigon` and `lon`
+to enable GDB to handle real `SIGSEGV` (e.g., while printing) and to stop other
+threads.
+
+`runloop` re-runs a program until there is a crash (useful for reproducing rare
+race conditions).
+
+Furthermore, there are some useful GDB extensions under `/c7/gdb/gdb_stm.py`
+that allow for inspecting segment-local pointers. To enable them, add the
+following line to your `.gdbinit`:
+
+    python exec(open('PATH-TO-STMGC/c7/gdb/gdb_stm.py').read())
+    
+
+
+
+## Building PyPy-STM
+
+The STM branch of PyPy contains a *copy* of the STMGC library. After changes to
+STMGC, run the `import_stmgc.py` script in `/rpython/translator/stm/`. In the
+following, `/` is the root of your PyPy checkout.
+
+ 0. Follow the [build instructions](http://doc.pypy.org/en/latest/build.html)
+    for PyPy until you get to the point to run the translation.
+
+ 1. The Makefile expects a `gcc-seg-gs` executable to be on the `$PATH`. This
+    should be a GCC that is either patched or a wrapper to GCC 6.1 that passes
+    the necessary options. In my case, this is a script that points to my custom
+    build of GCC with the following content:
+    
+        :::bash
+        #!/bin/bash
+        BUILD=/home/remi/work/bin/gcc-build
+        exec $BUILD/gcc/xgcc -B $BUILD/gcc -fno-ivopts -fno-tree-vectorize -fno-tree-loop-distribute-patterns "$@"
+    
+    
+ 2. `cd /pypy/goal/`
+ 
+ 3. A script to translate PyPy-STM (adapt all paths):
+ 
+        :::bash
+        #!/bin/bash
+        export PYPY_USESSION_KEEP=200
+        export PYPY_USESSION_DIR=~/pypy-usession
+        
+        STM=--stm #--stm
+        JIT=-Ojit #-Ojit #-O2
+        VERSION=$(hg id -i)
+        ionice -c3 pypy ~/pypy_dir/rpython/bin/rpython --no-shared --source $STM $JIT  targetpypystandalone.py
+        # --no-allworkingmodules
+        
+        notify-send "PyPy" "C source generated."
+        
+        cd ~/pypy-usession/usession-$(hg branch)-remi/testing_1/
+        ionice -c3 make -Bj4
+        
+        TIME=$(date +%y-%m-%d-%H:%M)
+        cp pypy-c ~/pypy_dir/pypy/goal/pypy-c-$STM-$JIT-$VERSION-$TIME
+        cp pypy-c ~/pypy_dir/pypy/goal/pypy-c
+        
+        notify-send "PyPy" "Make finished."
+    
+    The usession-folder will keep the produced C source files. You will need
+    them whenever you do a change to the STMGC library only (no need to
+    retranslate the full PyPy). In that case:
+    
+     1. Go to `~/pypy-usession/usession-stmgc-c8-$USER/testing_1/`
+     2. `make clean && make -j8` will rebuild all C sources
+        
+        Faster alternative that works in most cases: `rm ../module_cache/*.o`
+        instead of `make clean`. This will remove the compiled STMGC files,
+        forcing a rebuild from the *copy* in the `/rpython/translator/stm`
+        folder.
+        
+ 4. The script puts a `pypy-c` into `/pypy/goal/` that should be ready to run.
+
+
+### Log tools
+
+STMGC produces an event-log, if requested. Some tools to parse and analyse these
+logs are in the PyPy repository under `/pypy/stm/`. To produce a log, set the
+environment variable `PYPYSTM` to a file name. E.g.:
+
+`env PYPYSTM=log.pypystm pypy-c program.py`
+
+and then see some statistics with 
+
+`/pypy/stm/print_stm_log.py log.pypystm`
+
+
+### Benchmarks
+
+In PyPy's benchmark repository (`https://bitbucket.org/pypy/benchmarks`) under
+`multithread` is a collection of multi-threaded Python programs to measure
+performance.
+
+One way to run them is to check out the branch `multithread-runner` and do the
+following:
+
+`./runner.py pypy-c config-raytrace.json result.json`
+
+This will use the configuration in the JSON file and run a few iterations; then
+write the result into a JSON file again. It will also print the command-line
+used to run the benchmark, in case you don't want to use the runner. The
+`getresults.py` script can be used to compare two versions of PyPy against each
+other, but it is very limited.
+
+
+
+
diff --git a/c8/TODO b/c8/TODO
--- a/c8/TODO
+++ b/c8/TODO
@@ -1,3 +1,103 @@
+
+- investigate if userfaultfd() helps:
+  http://kernelnewbies.org/Linux_4.3#head-3deefea7b0add8c1b171b0e72ce3b69c5ed35cb0
+
+  AFAICS, we could avoid the current in-kernel-pagefault+SIGSEGV-handler for
+  making pages accessible on-demand, and replace that mechanism with a
+  user-pagefault "handler". That should save on the number of needed VMAs and
+  possibly be faster (although I'm quite unsure of that).
+
+- investigate if membarrier() helps:
+  http://man7.org/linux/man-pages/man2/membarrier.2.html
+
+
+##################
+Issue: lee_router_tm in one of its versions uses a temporary grid (big list)
+to do some calculations. This grid gets cleared everytime the router does
+one transaction (lay_next_track). Hence, the transaction writes a big amount
+of *old* memory.
+ * For one, clearing an array causes tons of stm_write_card() and creating
+   tons of small backup slices.
+ * Also, all of this stuff goes to the commit log, as we "modify" an old
+   object.
+ * Of course this is all completely unecessary, as the temporary grid is
+   basically thread-local, always gets cleared at the start of an atomic
+   block ("application-level transaction"), and therefore wouldn't need
+   to be mentioned in the commit log (and in theory wouldn't even need
+   reverting on abort).
+
+Here is a slice of the perf-profile:
+-   Total     Self
+-   14.39%    13.62%  pypy-c-clcollec  libc-2.19.so        [.] __memcpy_sse2_unaligned
+   - __memcpy_sse2_unaligned
+      + 80.47% import_objects.constprop.50
+      + 10.72% make_bk_slices_for_range
++   16.82%     1.43%  pypy-c-clcollec  pypy-c-clcollector  [.] make_bk_slices
++   14.71%     4.44%  pypy-c-clcollec  pypy-c-clcollector  [.] make_bk_slices_for_range
++    4.63%     4.62%  pypy-c-clcollec  pypy-c-clcollector  [.] go_to_the_past
+
+On this benchmark, pypy-stm is ~4x slower than pypy-default. It also doesn't
+scale at all (probably because a lot of the things above are actually protected
+by the privatization locks, which seem to be quite contended).
+Probably around 10% of the time is spent importing the changes done to
+a thread-local object. So here are a few ideas:
+ * Play with card-marking card size to speed up make_bk_slices?
+ * After marking a significant percentage of cards of an obj, maybe just
+   mark all of them (do a full write barrier)?
+ * Should modification-heavy transactions run longer (with high priority)?
+ * Special allocation area for thread-local objects which is always mapped
+   shared between all segments.
+    + changes to objs in this area do not need to be added to the commit log
+    - need to guarantee that only one thread/segment at a time accesses an
+      obj at a time
+    - needs explicit designation as thread-local at allocation time (by programmer)
+    - doesn't avoid creating backup copies
+ * Special objs that are transaction-local
+    + no need for backup copies, should always "reset" automatically
+    - needs programmer support
+    - limited applicability
+ * As long as a slice is in a page that is only mapped in one segment,
+   we can depend on the slice not being imported in other segs since
+   the page is not mapped. At least it avoids importing to seg0 before
+   major GC and also pushing overflow objs to seg0 on commit. However,
+   this requires the elimination of seg0 as it is now, which is hard:
+    * main concern is that major GC needs to trace objs somewhere, and right
+      now it traces them in seg0. We probably need a way to tell in which seg
+      an obj is accessible.
+    * one way would be to say an obj is always fully accessible in the seg it
+      was first allocated in, but that makes page resharing more difficult.
+      Still, we could record this information in the largemalloc-header, which
+      we first need to find (per obj check for where obj-header is accessible).
+      A check if all pages of an obj are accessible in some segment is probably
+      too slow, as is checking on-the-fly during tracing...
+    * largemalloc currently keeps its data structures in seg0. We would need
+      to keep the data structure up-to-date in all segments (e.g. by always
+      writing to all accessible segments, but this requires us to have the
+      privatization readlock when allocating, and privatization locks are
+      already contended in some cases)
+    * smallmalloc is a bit simpler since objs are always in a single page.
+      But I guess we still need to find the accessible page for each obj...
+  Overall:
+   + modifications to single-mapped pages would not be copied if they stay
+     single-mapped
+   + fully automatic/transparent
+   - provides only page-level granularity
+   - potentially slow complication of largemalloc and major GC
+   - requires us to put effort into making threads always run in the same
+     segment to increase effectivness
+   - doesn't avoid creating backup copies
+ * One could also argue that keeping around a tempgrid is not good and
+   the programmer should just re-create it in every transaction
+   (counter-intuitive). This indeed speeds up the benchmark ("only" 2x
+   slower than pypy-default), but causes tons of major GCs. These
+   major GCs completely prevent scaling, as they are stop-the-world.
+   So this opens up a whole new can of worms (concurrent, parallel,
+   incremental GC?).
+
+
+##################
+
+
 - stm_identityhash spends a good time figuring out if an obj is prebuilt
   (40% of its time). maybe after setup_prebuilt, we could defer the test
   of GCFLAG_HAS_SHADOW in id_or_identityhash to after an address comparison.
diff --git a/c8/demo/Makefile b/c8/demo/Makefile
--- a/c8/demo/Makefile
+++ b/c8/demo/Makefile
@@ -2,9 +2,9 @@
 # Makefile for the demos.
 #
 
-DEBUG_EXE = debug-demo2
-BUILD_EXE = build-demo2
-RELEASE_EXE = release-demo2
+DEBUG_EXE = debug-demo_simple
+BUILD_EXE = build-demo_simple
+RELEASE_EXE = release-demo_simple
 
 debug: $(DEBUG_EXE)       # with prints and asserts
 build: $(BUILD_EXE)       # without prints, but with asserts
diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h
--- a/c8/stm/atomic.h
+++ b/c8/stm/atomic.h
@@ -24,16 +24,16 @@
 
 #if defined(__i386__) || defined(__amd64__)
 
-  static inline void spin_loop(void) { asm("pause" : : : "memory"); }
-  static inline void write_fence(void) { asm("" : : : "memory"); }
+  static inline void stm_spin_loop(void) { asm("pause" : : : "memory"); }
+  static inline void stm_write_fence(void) { asm("" : : : "memory"); }
 /*# define atomic_exchange(ptr, old, new)  do {         \
           (old) = __sync_lock_test_and_set(ptr, new);   \
       } while (0)*/
 
 #else
 
-  static inline void spin_loop(void) { asm("" : : : "memory"); }
-  static inline void write_fence(void) { __sync_synchronize(); }
+  static inline void stm_spin_loop(void) { asm("" : : : "memory"); }
+  static inline void stm_write_fence(void) { __sync_synchronize(); }
 
 /*# define atomic_exchange(ptr, old, new)  do {           \
           (old) = *(ptr);                                 \
@@ -42,19 +42,19 @@
 #endif
 
 
-static inline void _spinlock_acquire(uint8_t *plock) {
+static inline void _stm_spinlock_acquire(uint8_t *plock) {
  retry:
     if (__builtin_expect(__sync_lock_test_and_set(plock, 1) != 0, 0)) {
-        spin_loop();
+        stm_spin_loop();
         goto retry;
     }
 }
-static inline void _spinlock_release(uint8_t *plock) {
+static inline void _stm_spinlock_release(uint8_t *plock) {
     assert(*plock == 1);
     __sync_lock_release(plock);
 }
-#define spinlock_acquire(lock) _spinlock_acquire(&(lock))
-#define spinlock_release(lock) _spinlock_release(&(lock))
+#define stm_spinlock_acquire(lock) _stm_spinlock_acquire(&(lock))
+#define stm_spinlock_release(lock) _stm_spinlock_release(&(lock))
 
 
 #endif  /* _STM_ATOMIC_H */
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 char *stm_object_pages;
@@ -113,212 +114,6 @@
 }
 
 
-/* ############# signal handler ############# */
-
-static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum,
-                                      bool only_if_not_modified)
-{
-    /* looks at all bk copies of objects overlapping page 'pagenum' and
-       copies the part in 'pagenum' back to the current segment */
-    dprintf(("copy_bk_objs_in_page_from(%d, %ld, %d)\n",
-             from_segnum, (long)pagenum, only_if_not_modified));
-
-    assert(modification_lock_check_rdlock(from_segnum));
-    struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects;
-    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
-    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
-
-    import_objects(only_if_not_modified ? -2 : -1,
-                   pagenum, undo, end);
-}
-
-static void go_to_the_past(uintptr_t pagenum,
-                           struct stm_commit_log_entry_s *from,
-                           struct stm_commit_log_entry_s *to)
-{
-    assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
-    assert(from->rev_num >= to->rev_num);
-    /* walk BACKWARDS the commit log and update the page 'pagenum',
-       initially at revision 'from', until we reach the revision 'to'. */
-
-    /* XXXXXXX Recursive algo for now, fix this! */
-    if (from != to) {
-        struct stm_commit_log_entry_s *cl = to->next;
-        go_to_the_past(pagenum, from, cl);
-
-        struct stm_undo_s *undo = cl->written;
-        struct stm_undo_s *end = cl->written + cl->written_count;
-
-        import_objects(-1, pagenum, undo, end);
-    }
-}
-
-
-long ro_to_acc = 0;
-static void handle_segfault_in_page(uintptr_t pagenum)
-{
-    /* assumes page 'pagenum' is ACCESS_NONE, privatizes it,
-       and validates to newest revision */
-    dprintf(("handle_segfault_in_page(%lu), seg %d\n", pagenum, STM_SEGMENT->segment_num));
-
-    /* XXX: bad, but no deadlocks: */
-    acquire_all_privatization_locks();
-
-    long i;
-    int my_segnum = STM_SEGMENT->segment_num;
-
-    uint8_t page_status = get_page_status_in(my_segnum, pagenum);
-    assert(page_status == PAGE_NO_ACCESS
-           || page_status == PAGE_READONLY);
-
-    if (page_status == PAGE_READONLY) {
-        /* make our page write-ready */
-        page_mark_accessible(my_segnum, pagenum);
-
-        dprintf((" > found READONLY, make others NO_ACCESS\n"));
-        /* our READONLY copy *has* to have the current data, no
-           copy necessary */
-        /* make READONLY pages in other segments NO_ACCESS */
-        for (i = 1; i < NB_SEGMENTS; i++) {
-            if (i == my_segnum)
-                continue;
-
-            if (get_page_status_in(i, pagenum) == PAGE_READONLY)
-                page_mark_inaccessible(i, pagenum);
-        }
-
-        ro_to_acc++;
-
-        release_all_privatization_locks();
-        return;
-    }
-
-    /* find who has the most recent revision of our page */
-    /* XXX: uh, *more* recent would be enough, right? */
-    int copy_from_segnum = -1;
-    uint64_t most_recent_rev = 0;
-    bool was_readonly = false;
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        if (i == my_segnum)
-            continue;
-
-        if (!was_readonly && get_page_status_in(i, pagenum) == PAGE_READONLY) {
-            was_readonly = true;
-            break;
-        }
-
-        struct stm_commit_log_entry_s *log_entry;
-        log_entry = get_priv_segment(i)->last_commit_log_entry;
-        if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS
-            && (copy_from_segnum == -1 || log_entry->rev_num > most_recent_rev)) {
-            copy_from_segnum = i;
-            most_recent_rev = log_entry->rev_num;
-        }
-    }
-    OPT_ASSERT(copy_from_segnum != my_segnum);
-
-    if (was_readonly) {
-        assert(page_status == PAGE_NO_ACCESS);
-        /* this case could be avoided by making all NO_ACCESS to READONLY
-           when resharing pages (XXX: better?).
-           We may go from NO_ACCESS->READONLY->ACCESSIBLE on write with
-           2 SIGSEGV in a row.*/
-        dprintf((" > make a previously NO_ACCESS page READONLY\n"));
-        page_mark_readonly(my_segnum, pagenum);
-
-        release_all_privatization_locks();
-        return;
-    }
-
-    /* make our page write-ready */
-    page_mark_accessible(my_segnum, pagenum);
-
-    /* account for this page now: XXX */
-    /* increment_total_allocated(4096); */
-
-
-    if (copy_from_segnum == -1) {
-        dprintf((" > found newly allocated page: copy from seg0\n"));
-
-        /* this page is only accessible in the sharing segment seg0 so far (new
-           allocation). We can thus simply mark it accessible here. */
-        pagecopy(get_virtual_page(my_segnum, pagenum),
-                 get_virtual_page(0, pagenum));
-        release_all_privatization_locks();
-        return;
-    }
-
-    dprintf((" > import data from seg %d\n", copy_from_segnum));
-
-    /* before copying anything, acquire modification locks from our and
-       the other segment */
-    uint64_t to_lock = (1UL << copy_from_segnum);
-    acquire_modification_lock_set(to_lock, my_segnum);
-    pagecopy(get_virtual_page(my_segnum, pagenum),
-             get_virtual_page(copy_from_segnum, pagenum));
-
-    /* if there were modifications in the page, revert them. */
-    copy_bk_objs_in_page_from(copy_from_segnum, pagenum, false);
-
-    /* we need to go from 'src_version' to 'target_version'.  This
-       might need a walk into the past. */
-    struct stm_commit_log_entry_s *src_version, *target_version;
-    src_version = get_priv_segment(copy_from_segnum)->last_commit_log_entry;
-    target_version = STM_PSEGMENT->last_commit_log_entry;
-
-
-    dprintf(("handle_segfault_in_page: rev %lu to rev %lu\n",
-             src_version->rev_num, target_version->rev_num));
-    /* adapt revision of page to our revision:
-       if our rev is higher than the page we copy from, everything
-       is fine as we never read/modified the page anyway
-     */
-    if (src_version->rev_num > target_version->rev_num)
-        go_to_the_past(pagenum, src_version, target_version);
-
-    release_modification_lock_set(to_lock, my_segnum);
-    release_all_privatization_locks();
-}
-
-static void _signal_handler(int sig, siginfo_t *siginfo, void *context)
-{
-    assert(_stm_segfault_expected > 0);
-
-    int saved_errno = errno;
-    char *addr = siginfo->si_addr;
-    dprintf(("si_addr: %p\n", addr));
-    if (addr == NULL || addr < stm_object_pages ||
-        addr >= stm_object_pages+TOTAL_MEMORY) {
-        /* actual segfault, unrelated to stmgc */
-        fprintf(stderr, "Segmentation fault: accessing %p\n", addr);
-        detect_shadowstack_overflow(addr);
-        abort();
-    }
-
-    int segnum = get_segment_of_linear_address(addr);
-    OPT_ASSERT(segnum != 0);
-    if (segnum != STM_SEGMENT->segment_num) {
-        fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from"
-                " seg %d\n", addr, segnum, STM_SEGMENT->segment_num);
-        abort();
-    }
-    dprintf(("-> segment: %d\n", segnum));
-
-    char *seg_base = STM_SEGMENT->segment_base;
-    uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL;
-    if (pagenum < END_NURSERY_PAGE) {
-        fprintf(stderr, "Segmentation fault: accessing %p (seg %d "
-                        "page %lu)\n", addr, segnum, pagenum);
-        abort();
-    }
-
-    DEBUG_EXPECT_SEGFAULT(false);
-    handle_segfault_in_page(pagenum);
-    DEBUG_EXPECT_SEGFAULT(true);
-
-    errno = saved_errno;
-    /* now return and retry */
-}
 
 /* ############# commit log ############# */
 
@@ -361,6 +156,7 @@
 }
 
 static void reset_modified_from_backup_copies(int segment_num, object_t *only_obj);  /* forward */
+static void undo_modifications_to_single_obj(int segment_num, object_t *only_obj); /* forward */
 
 static bool _stm_validate(void)
 {
@@ -442,7 +238,7 @@
                     for (; undo < end; undo++) {
                         object_t *obj;
 
-                        if (undo->type != TYPE_POSITION_MARKER) {
+                        if (LIKELY(undo->type != TYPE_POSITION_MARKER)) {
                             /* common case: 'undo->object' was written to
                                in this past commit, so we must check that
                                it was not read by us. */
@@ -472,13 +268,17 @@
                                an abort. However, from now on, we also assume
                                that an abort would not roll-back to what is in
                                the backup copy, as we don't trace the bkcpy
-                               during major GCs.
+                               during major GCs. (Seg0 may contain the version
+                               found in the other segment and thus not have to
+                               content of our bk_copy)
+
                                We choose the approach to reset all our changes
                                to this obj here, so that we can throw away the
                                backup copy completely: */
                             /* XXX: this browses through the whole list of modified
                                fragments; this may become a problem... */
-                            reset_modified_from_backup_copies(my_segnum, obj);
+                            undo_modifications_to_single_obj(my_segnum, obj);
+
                             continue;
                         }
 
@@ -604,11 +404,15 @@
 */
 static void _validate_and_attach(struct stm_commit_log_entry_s *new)
 {
+    uintptr_t cle_length = 0;
     struct stm_commit_log_entry_s *old;
 
     OPT_ASSERT(new != NULL);
     OPT_ASSERT(new != INEV_RUNNING);
 
+    cle_length = list_count(STM_PSEGMENT->modified_old_objects);
+    assert(cle_length == new->written_count * 3);
+
     soon_finished_or_inevitable_thread_segment();
 
  retry_from_start:
@@ -617,6 +421,16 @@
         stm_abort_transaction();
     }
 
+    if (cle_length != list_count(STM_PSEGMENT->modified_old_objects)) {
+        /* something changed the list of modified objs during _stm_validate; or
+         * during a major GC that also does _stm_validate(). That "something"
+         * can only be a reset of a noconflict obj. Thus, we recreate the CL
+         * entry */
+        free_cle(new);
+        new = _create_commit_log_entry();
+        cle_length = list_count(STM_PSEGMENT->modified_old_objects);
+    }
+
 #if STM_TESTS
     if (STM_PSEGMENT->transaction_state != TS_INEVITABLE
         && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
@@ -815,6 +629,10 @@
     size_t start_offset;
     if (first_call) {
         start_offset = 0;
+
+        /* flags like a never-touched obj */
+        assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+        assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
     } else {
         start_offset = -1;
     }
@@ -1249,8 +1067,7 @@
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
-    assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
-    assert(STM_PSEGMENT->finalizers == NULL);
+    assert(list_is_empty(STM_PSEGMENT->young_objects_with_destructors));
     assert(STM_PSEGMENT->active_queues == NULL);
 #ifndef NDEBUG
     /* this should not be used when objects_pointing_to_nursery == NULL */
@@ -1259,6 +1076,13 @@
 
     check_nursery_at_transaction_start();
 
+    if (tl->mem_reset_on_abort) {
+        assert(!!tl->mem_stored_for_reset_on_abort);
+        memcpy(tl->mem_stored_for_reset_on_abort, tl->mem_reset_on_abort,
+               tl->mem_bytes_to_reset_on_abort);
+    }
+
+
     /* Change read-version here, because if we do stm_validate in the
        safe-point below, we should not see our old reads from the last
        transaction. */
@@ -1282,7 +1106,7 @@
 }
 
 #ifdef STM_NO_AUTOMATIC_SETJMP
-static int did_abort = 0;
+int did_abort = 0;
 #endif
 
 long _stm_start_transaction(stm_thread_local_t *tl)
@@ -1294,6 +1118,12 @@
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
+    if (repeat_count) {
+        /* only if there was an abort, we need to reset the memory: */
+        if (tl->mem_reset_on_abort)
+            memcpy(tl->mem_reset_on_abort, tl->mem_stored_for_reset_on_abort,
+                   tl->mem_bytes_to_reset_on_abort);
+    }
     _do_start_transaction(tl);
 
     if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
@@ -1419,6 +1249,7 @@
     push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
 
+
     /* before releasing _stm_detached_inevitable_from_thread, perform
        the commit. Otherwise, the same thread whose (inev) transaction we try
        to commit here may start a new one in another segment *but* w/o
@@ -1436,7 +1267,7 @@
         /* but first, emit commit-event of this thread: */
         timing_event(STM_SEGMENT->running_thread, STM_TRANSACTION_COMMIT);
         STM_SEGMENT->running_thread = NULL;
-        write_fence();
+        stm_write_fence();
         assert(_stm_detached_inevitable_from_thread == -1);
         _stm_detached_inevitable_from_thread = 0;
     }
@@ -1481,6 +1312,48 @@
         invoke_general_finalizers(tl);
 }
 
+static void undo_modifications_to_single_obj(int segment_num, object_t *obj)
+{
+    /* special function used for noconflict objs to reset all their
+     * modifications and make them appear untouched in the current transaction.
+     * I.e., reset modifications and remove from all lists. */
+
+    struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
+
+    reset_modified_from_backup_copies(segment_num, obj);
+
+    /* reset read marker (must not be considered read either) */
+    ((struct stm_read_marker_s *)
+     (pseg->pub.segment_base + (((uintptr_t)obj) >> 4)))->rm = 0;
+
+    /* reset possibly marked cards */
+    if (get_page_status_in(segment_num, (uintptr_t)obj / 4096) == PAGE_ACCESSIBLE
+        && obj_should_use_cards(pseg->pub.segment_base, obj)) {
+        /* if header is not accessible, we didn't mark any cards */
+        _reset_object_cards(pseg, obj, CARD_CLEAR, false, false);
+    }
+
+    /* remove from all other lists */
+    LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
+       {
+           if (item == obj) {
+               /* copy last element over this one (HACK) */
+               _lst->count -= 1;
+               _lst->items[_i] = _lst->items[_lst->count];
+               break;
+           }
+       });
+    LIST_FOREACH_R(pseg->objects_pointing_to_nursery, object_t * /*item*/,
+       {
+           if (item == obj) {
+               /* copy last element over this one (HACK) */
+               _lst->count -= 1;
+               _lst->items[_i] = _lst->items[_lst->count];
+               break;
+           }
+       });
+}
+
 static void reset_modified_from_backup_copies(int segment_num, object_t *only_obj)
 {
 #pragma push_macro("STM_PSEGMENT")
@@ -1490,6 +1363,9 @@
     assert(modification_lock_check_wrlock(segment_num));
     DEBUG_EXPECT_SEGFAULT(false);
 
+    /* WARNING: resetting the obj will remove the WB flag. Make sure you either
+     * re-add it or remove it from lists where it was added based on the flag. */
+
     struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
     struct list_s *list = pseg->modified_old_objects;
     struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
@@ -1500,7 +1376,7 @@
             continue;
 
         object_t *obj = undo->object;
-        if (only_obj != NULL && obj != only_obj)
+        if (UNLIKELY(only_obj != NULL) && LIKELY(obj != only_obj))
             continue;
 
         char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj);
@@ -1515,19 +1391,15 @@
 
         free_bk(undo);
 
-        if (only_obj != NULL) {
-            assert(IMPLY(only_obj != NULL,
-                         (((struct object_s *)dst)->stm_flags
-                          & (GCFLAG_NO_CONFLICT
-                             | GCFLAG_WRITE_BARRIER
-                             | GCFLAG_WB_EXECUTED))
-                         == (GCFLAG_NO_CONFLICT | GCFLAG_WRITE_BARRIER)));
+        if (UNLIKELY(only_obj != NULL)) {
+            assert(((struct object_s *)dst)->stm_flags & GCFLAG_NO_CONFLICT);
+
             /* copy last element over this one */
             end--;
             list->count -= 3;
-            if (undo < end)
-                *undo = *end;
-            undo--;  /* next itr */
+            *undo = *end;
+            /* to neutralise the increment for the next iter: */
+            undo--;
         }
     }
 
@@ -1652,6 +1524,13 @@
 
     if (tl->mem_clear_on_abort)
         memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
+    if (tl->mem_reset_on_abort) {
+        /* temporarily set the memory of mem_reset_on_abort to zeros since in the
+           case of vmprof, the old value is really wrong if we didn't do the longjmp
+           back yet (that restores the C stack). We restore the memory in
+           _stm_start_transaction() */
+        memset(tl->mem_reset_on_abort, 0, tl->mem_bytes_to_reset_on_abort);
+    }
 
     invoke_and_clear_user_callbacks(1);   /* for abort */
 
@@ -1760,7 +1639,7 @@
        0. We have to wait for this to happen bc. otherwise, eg.
        _stm_detach_inevitable_transaction is not safe to do yet */
     while (_stm_detached_inevitable_from_thread == -1)
-        spin_loop();
+        stm_spin_loop();
     assert(_stm_detached_inevitable_from_thread == 0);
 
     soon_finished_or_inevitable_thread_segment();
@@ -1830,13 +1709,13 @@
     assert(STM_PSEGMENT->privatization_lock);
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
     assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+    assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
 
     ssize_t obj_size = stmcb_size_rounded_up(
         (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
     OPT_ASSERT(obj_size >= 16);
 
     if (LIKELY(is_small_uniform(obj))) {
-        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
         OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
         _synchronize_fragment((stm_char *)obj, obj_size);
         return;
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -1,3 +1,9 @@
+#ifndef _STMGC_H
+# error "must be compiled via stmgc.c"
+# include "../stmgc.h"  // silence flymake
+#endif
+
+
 #define _STM_CORE_H_
 
 #include <stdlib.h>
@@ -7,7 +13,8 @@
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
-
+#include <stdbool.h>
+#include "list.h"
 
 /************************************************************/
 
@@ -139,9 +146,9 @@
     pthread_t running_pthread;
 #endif
 
-    /* light finalizers */
-    struct list_s *young_objects_with_light_finalizers;
-    struct list_s *old_objects_with_light_finalizers;
+    /* destructors */
+    struct list_s *young_objects_with_destructors;
+    struct list_s *old_objects_with_destructors;
 
     /* regular finalizers (objs from the current transaction only) */
     struct finalizers_s *finalizers;
@@ -304,6 +311,14 @@
 static bool _stm_validate(void);
 static void _core_commit_transaction(bool external);
 
+static void import_objects(
+        int from_segnum,            /* or -1: from undo->backup,
+                                       or -2: from undo->backup if not modified */
+        uintptr_t pagenum,          /* or -1: "all accessible" */
+        struct stm_undo_s *undo,
+        struct stm_undo_s *end);
+
+
 static inline bool was_read_remote(char *base, object_t *obj)
 {
     uint8_t other_transaction_read_version =
@@ -326,12 +341,12 @@
 
 static inline void acquire_privatization_lock(int segnum)
 {
-    spinlock_acquire(get_priv_segment(segnum)->privatization_lock);
+    stm_spinlock_acquire(get_priv_segment(segnum)->privatization_lock);
 }
 
 static inline void release_privatization_lock(int segnum)
 {
-    spinlock_release(get_priv_segment(segnum)->privatization_lock);
+    stm_spinlock_release(get_priv_segment(segnum)->privatization_lock);
 }
 
 static inline bool all_privatization_locks_acquired(void)
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 #include <errno.h>
@@ -107,7 +108,7 @@
                is reset to a value different from -1 */
             dprintf(("reattach_transaction: busy wait...\n"));
             while (_stm_detached_inevitable_from_thread == -1)
-                spin_loop();
+                stm_spin_loop();
 
             /* then retry */
             goto restart;
@@ -157,7 +158,7 @@
         /* busy-loop: wait until _stm_detached_inevitable_from_thread
            is reset to a value different from -1 */
         while (_stm_detached_inevitable_from_thread == -1)
-            spin_loop();
+            stm_spin_loop();
         goto restart;
     }
     if (!__sync_bool_compare_and_swap(&_stm_detached_inevitable_from_thread,
@@ -209,7 +210,7 @@
         /* busy-loop: wait until _stm_detached_inevitable_from_thread
            is reset to a value different from -1 */
         while (_stm_detached_inevitable_from_thread == -1)
-            spin_loop();
+            stm_spin_loop();
         goto restart;
     }
 }
diff --git a/c8/stm/extra.c b/c8/stm/extra.c
--- a/c8/stm/extra.c
+++ b/c8/stm/extra.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -1,68 +1,100 @@
-
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
+#endif
+#include "finalizer.h"
+#include "fprintcolor.h"
+#include "nursery.h"
+#include "gcpage.h"
 
 /* callbacks */
-void (*stmcb_light_finalizer)(object_t *);
+void (*stmcb_destructor)(object_t *);
 void (*stmcb_finalizer)(object_t *);
 
 
 static void init_finalizers(struct finalizers_s *f)
 {
     f->objects_with_finalizers = list_create();
-    f->count_non_young = 0;
-    f->run_finalizers = NULL;
-    f->running_next = NULL;
+    f->probably_young_objects_with_finalizers = list_create();
+    f->run_finalizers = list_create();
+    f->lock = 0;
+    f->running_trigger_now = NULL;
 }
 
 static void setup_finalizer(void)
 {
     init_finalizers(&g_finalizers);
+
+    for (long j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+
+        assert(pseg->finalizers == NULL);
+        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
+        if (f == NULL)
+            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
+        init_finalizers(f);
+        pseg->finalizers = f;
+    }
 }
 
-static void teardown_finalizer(void)
+void stm_setup_finalizer_queues(int number, stm_finalizer_trigger_fn *triggers)
 {
-    if (g_finalizers.run_finalizers != NULL)
-        list_free(g_finalizers.run_finalizers);
-    list_free(g_finalizers.objects_with_finalizers);
+    assert(g_finalizer_triggers.count == 0);
+    assert(g_finalizer_triggers.triggers == NULL);
+
+    g_finalizer_triggers.count = number;
+    g_finalizer_triggers.triggers = (stm_finalizer_trigger_fn *)
+        malloc(number * sizeof(stm_finalizer_trigger_fn));
+
+    for (int qindex = 0; qindex < number; qindex++) {
+        g_finalizer_triggers.triggers[qindex] = triggers[qindex];
+        dprintf(("setup_finalizer_queue(qindex=%d,fun=%p)\n", qindex, triggers[qindex]));
+    }
+}
+
+static void teardown_finalizer(void) {
+    LIST_FREE(g_finalizers.run_finalizers);
+    LIST_FREE(g_finalizers.objects_with_finalizers);
+    LIST_FREE(g_finalizers.probably_young_objects_with_finalizers);
     memset(&g_finalizers, 0, sizeof(g_finalizers));
+
+    if (g_finalizer_triggers.triggers)
+        free(g_finalizer_triggers.triggers);
+    memset(&g_finalizer_triggers, 0, sizeof(g_finalizer_triggers));
 }
 
 static void _commit_finalizers(void)
 {
     /* move finalizer lists to g_finalizers for major collections */
     while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
-        spin_loop();
+        stm_spin_loop();
     }
 
-    if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
+    struct finalizers_s *local_fs = STM_PSEGMENT->finalizers;
+    if (!list_is_empty(local_fs->run_finalizers)) {
         /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
            'g_finalizers.run_finalizers', dropping any initial NULLs
            (finalizers already called) */
-        struct list_s *src = STM_PSEGMENT->finalizers->run_finalizers;
-        uintptr_t frm = 0;
-        if (STM_PSEGMENT->finalizers->running_next != NULL) {
-            frm = *STM_PSEGMENT->finalizers->running_next;
-            assert(frm <= list_count(src));
-            *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
-        }
-        if (frm < list_count(src)) {
-            if (g_finalizers.run_finalizers == NULL)
-                g_finalizers.run_finalizers = list_create();
+        struct list_s *src = local_fs->run_finalizers;
+        if (list_count(src)) {
             g_finalizers.run_finalizers = list_extend(
                 g_finalizers.run_finalizers,
-                src, frm);
+                src, 0);
         }
-        list_free(src);
     }
+    LIST_FREE(local_fs->run_finalizers);
 
     /* copy the whole 'STM_PSEGMENT->finalizers->objects_with_finalizers'
        into 'g_finalizers.objects_with_finalizers' */
     g_finalizers.objects_with_finalizers = list_extend(
         g_finalizers.objects_with_finalizers,
-        STM_PSEGMENT->finalizers->objects_with_finalizers, 0);
-    list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
+        local_fs->objects_with_finalizers, 0);
+    LIST_FREE(local_fs->objects_with_finalizers);
+    assert(list_is_empty(local_fs->probably_young_objects_with_finalizers));
+    LIST_FREE(local_fs->probably_young_objects_with_finalizers);
 
-    free(STM_PSEGMENT->finalizers);
-    STM_PSEGMENT->finalizers = NULL;
+    // re-init
+    init_finalizers(local_fs);
 
     __sync_lock_release(&g_finalizers.lock);
 }
@@ -71,24 +103,22 @@
 {
     /* like _commit_finalizers(), but forget everything from the
        current transaction */
-    if (pseg->finalizers != NULL) {
-        if (pseg->finalizers->run_finalizers != NULL) {
-            if (pseg->finalizers->running_next != NULL) {
-                *pseg->finalizers->running_next = (uintptr_t)-1;
-            }
-            list_free(pseg->finalizers->run_finalizers);
-        }
-        list_free(pseg->finalizers->objects_with_finalizers);
-        free(pseg->finalizers);
-        pseg->finalizers = NULL;
-    }
+    LIST_FREE(pseg->finalizers->run_finalizers);
+    LIST_FREE(pseg->finalizers->objects_with_finalizers);
+    LIST_FREE(pseg->finalizers->probably_young_objects_with_finalizers);
+    // re-init
+    init_finalizers(pseg->finalizers);
+
+    // if we were running triggers, release the lock:
+    if (g_finalizers.running_trigger_now == pseg)
+        g_finalizers.running_trigger_now = NULL;
 
     /* call the light finalizers for objects that are about to
        be forgotten from the current transaction */
     char *old_gs_register = STM_SEGMENT->segment_base;
     bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
 
-    struct list_s *lst = pseg->young_objects_with_light_finalizers;
+    struct list_s *lst = pseg->young_objects_with_destructors;
     long i, count = list_count(lst);
     if (lst > 0) {
         for (i = 0; i < count; i++) {
@@ -98,15 +128,15 @@
                 set_gs_register(pseg->pub.segment_base);
                 must_fix_gs = false;
             }
-            stmcb_light_finalizer(obj);
+            stmcb_destructor(obj);
         }
         list_clear(lst);
     }
 
     /* also deals with overflow objects: they are at the tail of
-       old_objects_with_light_finalizers (this list is kept in order
+       old_objects_with_destructors (this list is kept in order
        and we cannot add any already-committed object) */
-    lst = pseg->old_objects_with_light_finalizers;
+    lst = pseg->old_objects_with_destructors;
     count = list_count(lst);
     while (count > 0) {
         object_t *obj = (object_t *)list_item(lst, --count);
@@ -117,7 +147,7 @@
             set_gs_register(pseg->pub.segment_base);
             must_fix_gs = false;
         }
-        stmcb_light_finalizer(obj);
+        stmcb_destructor(obj);
     }
 
     if (STM_SEGMENT->segment_base != old_gs_register)
@@ -125,44 +155,42 @@
 }
 
 
-void stm_enable_light_finalizer(object_t *obj)
+void stm_enable_destructor(object_t *obj)
 {
     if (_is_young(obj)) {
-        LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->young_objects_with_destructors, obj);
     }
     else {
         assert(_is_from_same_transaction(obj));
-        LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_destructors, obj);
     }
 }
 
-object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
+
+void stm_enable_finalizer(int queue_index, object_t *obj)
 {
-    object_t *obj = _stm_allocate_external(size_rounded_up);
-
-    if (STM_PSEGMENT->finalizers == NULL) {
-        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
-        if (f == NULL)
-            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
-        init_finalizers(f);
-        STM_PSEGMENT->finalizers = f;
+    if (_is_young(obj)) {
+        LIST_APPEND(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers, queue_index);
     }
-    assert(STM_PSEGMENT->finalizers->count_non_young
-           <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers));
-    LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
-    return obj;
+    else {
+        assert(_is_from_same_transaction(obj));
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, queue_index);
+    }
 }
 
 
+
 /************************************************************/
-/*  Light finalizers
+/*  Destructors
 */
 
-static void deal_with_young_objects_with_finalizers(void)
+static void deal_with_young_objects_with_destructors(void)
 {
-    /* for light finalizers: executes finalizers for objs that don't survive
+    /* for destructors: executes destructors for objs that don't survive
        this minor gc */
-    struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
+    struct list_s *lst = STM_PSEGMENT->young_objects_with_destructors;
     long i, count = list_count(lst);
     for (i = 0; i < count; i++) {
         object_t *obj = (object_t *)list_item(lst, i);
@@ -171,28 +199,29 @@
         object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
         if (pforwarded_array[0] != GCWORD_MOVED) {
             /* not moved: the object dies */
-            stmcb_light_finalizer(obj);
+            stmcb_destructor(obj);
         }
         else {
             obj = pforwarded_array[1]; /* moved location */
             assert(!_is_young(obj));
-            LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+            LIST_APPEND(STM_PSEGMENT->old_objects_with_destructors, obj);
         }
     }
     list_clear(lst);
 }
 
-static void deal_with_old_objects_with_finalizers(void)
+static void deal_with_old_objects_with_destructors(void)
 {
-    /* for light finalizers */
+    /* for destructors */
     int old_gs_register = STM_SEGMENT->segment_num;
     int current_gs_register = old_gs_register;
     long j;
-    assert(list_is_empty(get_priv_segment(0)->old_objects_with_light_finalizers));
+    assert(list_is_empty(get_priv_segment(0)->old_objects_with_destructors));
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
 
-        struct list_s *lst = pseg->old_objects_with_light_finalizers;
+        assert(list_is_empty(pseg->young_objects_with_destructors));
+        struct list_s *lst = pseg->old_objects_with_destructors;
         long i, count = list_count(lst);
         lst->count = 0;
         for (i = 0; i < count; i++) {
@@ -214,7 +243,7 @@
                     set_gs_register(get_segment_base(j));
                     current_gs_register = j;
                 }
-                stmcb_light_finalizer(obj);
+                stmcb_destructor(obj);
             }
             else {
                 /* object survives */
@@ -227,6 +256,7 @@
 }
 
 
+
 /************************************************************/
 /*  Algorithm for regular (non-light) finalizers.
     Follows closely pypy/doc/discussion/finalizer-order.rst
@@ -325,20 +355,23 @@
 
     struct list_s *marked = list_create();
 
+    assert(list_is_empty(f->probably_young_objects_with_finalizers));
     struct list_s *lst = f->objects_with_finalizers;
     long i, count = list_count(lst);
     lst->count = 0;
-    f->count_non_young = 0;
 
-    for (i = 0; i < count; i++) {
+    for (i = 0; i < count; i += 2) {
         object_t *x = (object_t *)list_item(lst, i);
+        uintptr_t qindex = list_item(lst, i + 1);
 
         assert(_finalization_state(x) != 1);
         if (_finalization_state(x) >= 2) {
             list_set_item(lst, lst->count++, (uintptr_t)x);
+            list_set_item(lst, lst->count++, qindex);
             continue;
         }
         LIST_APPEND(marked, x);
+        LIST_APPEND(marked, qindex);
 
         struct list_s *pending = _finalizer_pending;
         LIST_APPEND(pending, x);
@@ -370,27 +403,29 @@
     struct list_s *run_finalizers = f->run_finalizers;
 
     long i, count = list_count(marked);
-    for (i = 0; i < count; i++) {
+    for (i = 0; i < count; i += 2) {
         object_t *x = (object_t *)list_item(marked, i);
+        uintptr_t qindex = list_item(marked, i + 1);
 
         int state = _finalization_state(x);
         assert(state >= 2);
         if (state == 2) {
-            if (run_finalizers == NULL)
-                run_finalizers = list_create();
             LIST_APPEND(run_finalizers, x);
+            LIST_APPEND(run_finalizers, qindex);
             _recursively_bump_finalization_state_from_2_to_3(pseg, x);
         }
         else {
             struct list_s *lst = f->objects_with_finalizers;
             list_set_item(lst, lst->count++, (uintptr_t)x);
+            list_set_item(lst, lst->count++, qindex);
         }
     }
-    list_free(marked);
+    LIST_FREE(marked);
 
     f->run_finalizers = run_finalizers;
 }
 
+
 static void deal_with_objects_with_finalizers(void)
 {
     /* for non-light finalizers */
@@ -433,11 +468,10 @@
 static void mark_visit_from_finalizer1(
     struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
-    if (f != NULL && f->run_finalizers != NULL) {
-        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
-                       ({
-                           mark_visit_possibly_overflow_object(item, pseg);
-                       }));
+    long i, count = list_count(f->run_finalizers);
+    for (i = 0; i < count; i += 2) {
+        object_t *x = (object_t *)list_item(f->run_finalizers, i);
+        mark_visit_possibly_overflow_object(x, pseg);
     }
 }
 
@@ -451,40 +485,6 @@
     mark_visit_from_finalizer1(get_priv_segment(0), &g_finalizers);
 }
 
-static void _execute_finalizers(struct finalizers_s *f)
-{
-    if (f->run_finalizers == NULL)
-        return;   /* nothing to do */
-
- restart:
-    if (f->running_next != NULL)
-        return;   /* in a nested invocation of execute_finalizers() */
-
-    uintptr_t next = 0, total = list_count(f->run_finalizers);
-    f->running_next = &next;
-
-    while (next < total) {
-        object_t *obj = (object_t *)list_item(f->run_finalizers, next);
-        list_set_item(f->run_finalizers, next, 0);
-        next++;
-
-        stmcb_finalizer(obj);
-    }
-    if (next == (uintptr_t)-1) {
-        /* transaction committed: the whole 'f' was freed */
-        return;
-    }
-    f->running_next = NULL;
-
-    if (f->run_finalizers->count > total) {
-        memmove(f->run_finalizers->items,
-                f->run_finalizers->items + total,
-                (f->run_finalizers->count - total) * sizeof(uintptr_t));
-        goto restart;
-    }
-
-    LIST_FREE(f->run_finalizers);
-}
 
 /* XXX: according to translator.backendopt.finalizer, getfield_gc
         for primitive types is a safe op in light finalizers.
@@ -492,43 +492,185 @@
         getfield on *dying obj*).
 */
 
+static void _trigger_finalizer_queues(struct finalizers_s *f)
+{
+    /* runs triggers of finalizer queues that have elements in the queue. May
+       NOT run outside of a transaction, but triggers never leave the
+       transactional zone.
+
+       returns true if there are also old-style finalizers to run */
+    assert(in_transaction(STM_PSEGMENT->pub.running_thread));
+
+    bool *to_trigger = (bool*)alloca(g_finalizer_triggers.count * sizeof(bool));
+    memset(to_trigger, 0, g_finalizer_triggers.count * sizeof(bool));
+
+    while (__sync_lock_test_and_set(&f->lock, 1) != 0) {
+        /* somebody is adding more finalizers (_commit_finalizer()) */
+        stm_spin_loop();
+    }
+
+    int count = list_count(f->run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(f->run_finalizers, i + 1);
+        dprintf(("qindex=%d\n", qindex));
+        to_trigger[qindex] = true;
+    }
+
+    __sync_lock_release(&f->lock);
+
+    // trigger now:
+    for (int i = 0; i < g_finalizer_triggers.count; i++) {
+        if (to_trigger[i]) {
+            dprintf(("invoke-finalizer-trigger(qindex=%d)\n", i));
+            g_finalizer_triggers.triggers[i]();
+        }
+    }
+}
+
+static bool _has_oldstyle_finalizers(struct finalizers_s *f)
+{
+    int count = list_count(f->run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(f->run_finalizers, i + 1);
+        if (qindex == -1)
+            return true;
+    }
+    return false;
+}
+
+static void _invoke_local_finalizers()
+{
+    /* called inside a transaction; invoke local triggers, process old-style
+     * local finalizers */
+    dprintf(("invoke_local_finalizers %lu\n", list_count(STM_PSEGMENT->finalizers->run_finalizers)));
+    if (list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)
+        && list_is_empty(g_finalizers.run_finalizers))
+        return;
+
+    struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
+    //try to run local triggers
+    if (STM_PSEGMENT->finalizers->running_trigger_now == NULL) {
+        // we are not recursively running them
+        STM_PSEGMENT->finalizers->running_trigger_now = pseg;
+        _trigger_finalizer_queues(STM_PSEGMENT->finalizers);
+        STM_PSEGMENT->finalizers->running_trigger_now = NULL;
+    }
+
+    // try to run global triggers
+    if (__sync_lock_test_and_set(&g_finalizers.running_trigger_now, pseg) == NULL) {
+        // nobody is already running these triggers (recursively)
+        _trigger_finalizer_queues(&g_finalizers);
+        g_finalizers.running_trigger_now = NULL;
+    }
+
+    if (!_has_oldstyle_finalizers(STM_PSEGMENT->finalizers))
+        return; // no oldstyle to run
+
+    object_t *obj;
+    while ((obj = stm_next_to_finalize(-1)) != NULL) {
+        stmcb_finalizer(obj);
+    }
+}
+
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
-    /* called between transactions */
+    /* called between transactions
+     * triggers not called here, since all should have been called already in _invoke_local_finalizers!
+     * run old-style finalizers (q_index=-1)
+     * queues that are not empty. */
+    dprintf(("invoke_general_finalizers %lu\n", list_count(g_finalizers.run_finalizers)));
+    if (list_is_empty(g_finalizers.run_finalizers))
+        return;
+
+    if (!_has_oldstyle_finalizers(&g_finalizers))
+        return; // no oldstyle to run
+
+    // run old-style finalizers:
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
     _stm_start_transaction(tl);
-    /* XXX: become inevitable, bc. otherwise, we would need to keep
-       around the original g_finalizers.run_finalizers to restore it
-       in case of an abort. */
-    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
-    /* did it work? */
-    if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {   /* no */
-        /* avoid blocking here, waiting for another INEV transaction.
-           If we did that, application code could not proceed (start the
-           next transaction) and it will not be obvious from the profile
-           why we were WAITing. */
-        _stm_commit_transaction();
-        stm_rewind_jmp_leaveframe(tl, &rjbuf);
-        return;
-    }
 
-    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
-        /* somebody is adding more finalizers (_commit_finalizer()) */
-        spin_loop();
-    }
-    struct finalizers_s copy = g_finalizers;
-    assert(copy.running_next == NULL);
-    g_finalizers.run_finalizers = NULL;
-    /* others may add to g_finalizers again: */
-    __sync_lock_release(&g_finalizers.lock);
-
-    if (copy.run_finalizers != NULL) {
-        _execute_finalizers(&copy);
+    dprintf(("invoke_oldstyle_finalizers %lu\n", list_count(g_finalizers.run_finalizers)));
+    object_t *obj;
+    while ((obj = stm_next_to_finalize(-1)) != NULL) {
+        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        stmcb_finalizer(obj);
     }
 
     _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);
+}
 
-    LIST_FREE(copy.run_finalizers);
+object_t* stm_next_to_finalize(int queue_index) {
+    assert(STM_PSEGMENT->transaction_state != TS_NONE);
+
+    /* first check local run_finalizers queue, then global */
+    if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) {
+        struct list_s *lst = STM_PSEGMENT->finalizers->run_finalizers;
+        int count = list_count(lst);
+        for (int i = 0; i < count; i += 2) {
+            int qindex = (int)list_item(lst, i + 1);
+            if (qindex == queue_index) {
+                /* no need to become inevitable for local ones */
+                /* Remove obj from list and return it. */
+                object_t *obj = (object_t*)list_item(lst, i);
+                int remaining = count - i - 2;
+                if (remaining > 0) {
+                    memmove(&lst->items[i],
+                            &lst->items[i + 2],
+                            remaining * sizeof(uintptr_t));
+                }
+                lst->count -= 2;
+                return obj;
+            }
+        }
+    }
+
+    /* no local finalizers found, continue in global list */
+
+    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
+        /* somebody is adding more finalizers (_commit_finalizer()) */
+        stm_spin_loop();
+    }
+
+    struct list_s *lst = g_finalizers.run_finalizers;
+    int count = list_count(lst);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(lst, i + 1);
+        if (qindex == queue_index) {
+            /* XXX: become inevitable, bc. otherwise, we would need to keep
+               around the original g_finalizers.run_finalizers to restore it
+               in case of an abort. */
+            if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {
+                _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+                /* did it work? */
+                if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {   /* no */
+                    /* avoid blocking here, waiting for another INEV transaction.
+                       If we did that, application code could not proceed (start the
+                       next transaction) and it will not be obvious from the profile
+                       why we were WAITing. XXX: still true? */
+                    __sync_lock_release(&g_finalizers.lock);
+                    return NULL;
+                }
+            }
+
+            /* Remove obj from list and return it. */
+            object_t *obj = (object_t*)list_item(lst, i);
+            int remaining = count - i - 2;
+            if (remaining > 0) {
+                memmove(&lst->items[i],
+                        &lst->items[i + 2],
+                        remaining * sizeof(uintptr_t));
+            }
+            lst->count -= 2;
+
+            __sync_lock_release(&g_finalizers.lock);
+            return obj;
+        }
+    }
+
+    /* others may add to g_finalizers again: */
+    __sync_lock_release(&g_finalizers.lock);
+
+    return NULL;
 }
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
--- a/c8/stm/finalizer.h
+++ b/c8/stm/finalizer.h
@@ -1,16 +1,20 @@
+#ifndef _STM_FINALIZER_H_
+#define _STM_FINALIZER_H_
+
+#include <stdint.h>
 
 /* see deal_with_objects_with_finalizers() for explanation of these fields */
 struct finalizers_s {
     long lock;
+    struct stm_priv_segment_info_s * running_trigger_now; /* our PSEG, if we are running triggers */
     struct list_s *objects_with_finalizers;
-    uintptr_t count_non_young;
+    struct list_s *probably_young_objects_with_finalizers; /* empty on g_finalizers! */
     struct list_s *run_finalizers;
-    uintptr_t *running_next;
 };
 
 static void mark_visit_from_finalizer_pending(void);
-static void deal_with_young_objects_with_finalizers(void);
-static void deal_with_old_objects_with_finalizers(void);
+static void deal_with_young_objects_with_destructors(void);
+static void deal_with_old_objects_with_destructors(void);
 static void deal_with_objects_with_finalizers(void);
 
 static void setup_finalizer(void);
@@ -27,19 +31,22 @@
 
 /* regular finalizers (objs from already-committed transactions) */
 static struct finalizers_s g_finalizers;
+static struct {
+    int count;
+    stm_finalizer_trigger_fn *triggers;
+} g_finalizer_triggers;
+
 
 static void _invoke_general_finalizers(stm_thread_local_t *tl);
+static void _invoke_local_finalizers(void);
 
 #define invoke_general_finalizers(tl)    do {   \
-    if (g_finalizers.run_finalizers != NULL)    \
-        _invoke_general_finalizers(tl);         \
+     _invoke_general_finalizers(tl);         \
 } while (0)
 
-static void _execute_finalizers(struct finalizers_s *f);
 
-#define any_local_finalizers() (STM_PSEGMENT->finalizers != NULL &&         \
-                               STM_PSEGMENT->finalizers->run_finalizers != NULL)
 #define exec_local_finalizers()  do {                   \
-    if (any_local_finalizers())                         \
-        _execute_finalizers(STM_PSEGMENT->finalizers);  \
+     _invoke_local_finalizers();                     \
 } while (0)
+
+#endif
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 #include <fcntl.h>           /* For O_* constants */
diff --git a/c8/stm/fprintcolor.h b/c8/stm/fprintcolor.h
--- a/c8/stm/fprintcolor.h
+++ b/c8/stm/fprintcolor.h
@@ -1,3 +1,7 @@
+#ifndef _FPRINTCOLOR_H
+#define _FPRINTCOLOR_H
+
+
 /* ------------------------------------------------------------ */
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
@@ -40,3 +44,5 @@
 __attribute__((unused))
 static void stm_fatalerror(const char *format, ...)
      __attribute__((format (printf, 1, 2), noreturn));
+
+#endif
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 static struct tree_s *tree_prebuilt_objs = NULL;     /* XXX refactor */
@@ -75,7 +76,7 @@
 
 
     /* uncommon case: need to initialize some more pages */
-    spinlock_acquire(lock_growth_large);
+    stm_spinlock_acquire(lock_growth_large);
 
     char *start = uninitialized_page_start;
     if (addr + size > start) {
@@ -99,7 +100,7 @@
 
     ((struct object_s*)addr)->stm_flags = 0;
 
-    spinlock_release(lock_growth_large);
+    stm_spinlock_release(lock_growth_large);
     return (stm_char*)(addr - stm_object_pages);
 }
 
@@ -188,7 +189,7 @@
     DEBUG_EXPECT_SEGFAULT(true);
     release_all_privatization_locks();
 
-    write_fence();     /* make sure 'nobj' is fully initialized from
+    stm_write_fence();     /* make sure 'nobj' is fully initialized from
                           all threads here */
     return (object_t *)nobj;
 }
@@ -976,9 +977,9 @@
 
     LIST_FREE(marked_objects_to_trace);
 
-    /* weakrefs and execute old light finalizers */
+    /* weakrefs and execute old destructors */
     stm_visit_old_weakrefs();
-    deal_with_old_objects_with_finalizers();
+    deal_with_old_objects_with_destructors();
 
     /* cleanup */
     clean_up_segment_lists();
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
--- a/c8/stm/gcpage.h
+++ b/c8/stm/gcpage.h
@@ -1,3 +1,7 @@
+#ifndef _STM_GCPAGE_H_
+#define _STM_GCPAGE_H_
+
+#include <stdbool.h>
 
 /* Granularity when grabbing more unused pages: take 20 at a time */
 #define GCPAGE_NUM_PAGES   20
@@ -22,3 +26,9 @@
 static void major_collection_with_mutex(void);
 static bool largemalloc_keep_object_at(char *data);   /* for largemalloc.c */
 static bool smallmalloc_keep_object_at(char *data);   /* for smallmalloc.c */
+
+static inline bool mark_visited_test(object_t *obj);
+static bool is_overflow_obj_safe(struct stm_priv_segment_info_s *pseg, object_t *obj);
+static void mark_visit_possibly_overflow_object(object_t *obj, struct stm_priv_segment_info_s *pseg);
+
+#endif
diff --git a/c8/stm/hash_id.c b/c8/stm/hash_id.c
--- a/c8/stm/hash_id.c
+++ b/c8/stm/hash_id.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c
--- a/c8/stm/hashtable.c
+++ b/c8/stm/hashtable.c
@@ -40,7 +40,7 @@
 
 Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
 */
-
+#include <stdint.h>
 
 uint32_t stm_hashtable_entry_userdata;
 
@@ -216,7 +216,7 @@
     }
     biggertable->resize_counter = rc;
 
-    write_fence();   /* make sure that 'biggertable' is valid here,
+    stm_write_fence();   /* make sure that 'biggertable' is valid here,
                         and make sure 'table->resize_counter' is updated
                         ('table' must be immutable from now on). */
     VOLATILE_HASHTABLE(hashtable)->table = biggertable;
@@ -278,7 +278,7 @@
        just now.  In both cases, this thread must simply spin loop.
     */
     if (IS_EVEN(rc)) {
-        spin_loop();
+        stm_spin_loop();
         goto restart;
     }
     /* in the other cases, we need to grab the RESIZING_LOCK.
@@ -348,7 +348,7 @@
             hashtable->additions++;
         }
         table->items[i] = entry;
-        write_fence();     /* make sure 'table->items' is written here */
+        stm_write_fence();     /* make sure 'table->items' is written here */
         VOLATILE_TABLE(table)->resize_counter = rc - 6;    /* unlock */
         stm_read((object_t*)entry);
         return entry;
@@ -437,7 +437,7 @@
     table = VOLATILE_HASHTABLE(hashtable)->table;
     rc = VOLATILE_TABLE(table)->resize_counter;
     if (IS_EVEN(rc)) {
-        spin_loop();
+        stm_spin_loop();
         goto restart;
     }
 
diff --git a/c8/stm/largemalloc.c b/c8/stm/largemalloc.c
--- a/c8/stm/largemalloc.c
+++ b/c8/stm/largemalloc.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 /* This contains a lot of inspiration from malloc() in the GNU C Library.
@@ -116,12 +117,12 @@
 
 static void lm_lock(void)
 {
-    spinlock_acquire(lm.lock);
+    stm_spinlock_acquire(lm.lock);
 }
 
 static void lm_unlock(void)
 {
-    spinlock_release(lm.lock);
+    stm_spinlock_release(lm.lock);
 }
 
 
diff --git a/c8/stm/list.c b/c8/stm/list.c
--- a/c8/stm/list.c
+++ b/c8/stm/list.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/list.h b/c8/stm/list.h
--- a/c8/stm/list.h
+++ b/c8/stm/list.h
@@ -1,5 +1,11 @@
+#ifndef _LIST_H
+#define _LIST_H
+
+
 #include <stdlib.h>
 #include <stdbool.h>
+#include <stdint.h>
+
 
 /************************************************************/
 
@@ -11,13 +17,13 @@
 
 static struct list_s *list_create(void) __attribute__((unused));
 
-static inline void list_free(struct list_s *lst)
+static inline void _list_free(struct list_s *lst)
 {
     free(lst);
 }
 
 #define LIST_CREATE(lst)  ((lst) = list_create())
-#define LIST_FREE(lst)  (list_free(lst), (lst) = NULL)
+#define LIST_FREE(lst)  (_list_free(lst), (lst) = NULL)
 
 
 static struct list_s *_list_grow(struct list_s *, uintptr_t);
@@ -245,3 +251,5 @@
     TREE_FIND(tree, addr, result, return false);
     return true;
 }
+
+#endif
diff --git a/c8/stm/marker.c b/c8/stm/marker.c
--- a/c8/stm/marker.c
+++ b/c8/stm/marker.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c

From pypy.commits at gmail.com  Mon Feb 27 11:40:13 2017
From: pypy.commits at gmail.com (Raemi)
Date: Mon, 27 Feb 2017 08:40:13 -0800 (PST)
Subject: [pypy-commit] stmgc c8-reshare-pages: do a direct
 NOACCES->ACCESSIBLE if the SIGSEGV is a write operation
Message-ID: <58b4566d.014b2e0a.3338c.ea13@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: c8-reshare-pages
Changeset: r2017:5911d9fa238e
Date: 2017-02-27 17:38 +0100
http://bitbucket.org/pypy/stmgc/changeset/5911d9fa238e/

Log:	do a direct NOACCES->ACCESSIBLE if the SIGSEGV is a write operation

	Apparently the kernel can tell us if the SIGSEGV is caused by a
	write. Then we can avoid the NOACCESS->RO->RW dance, saving ~10% of
	SIGSEGV on test_random.

	> strace ~/pypy_dir/pytest.py test_zrandom.py -vx -k 10 2>&1 |egrep
	"SIGSEGV"|wc -l

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -817,7 +817,7 @@
         if (get_page_status_in(my_segnum, page) != PAGE_ACCESSIBLE) {
             release_privatization_lock(STM_SEGMENT->segment_num);
             /* emulate pagefault -> PAGE_ACCESSIBLE/READONLY: */
-            handle_segfault_in_page(page);
+            handle_segfault_in_page(page, false);
             volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
             *dummy = *dummy;            /* force segfault (incl. writing) */
             acquire_privatization_lock(STM_SEGMENT->segment_num);
diff --git a/c8/stm/signal_handler.c b/c8/stm/signal_handler.c
--- a/c8/stm/signal_handler.c
+++ b/c8/stm/signal_handler.c
@@ -61,8 +61,26 @@
 }
 
 
+static void readonly_to_accessible(int my_segnum, uintptr_t pagenum)
+{
+    /* make our page write-ready */
+    page_mark_accessible(my_segnum, pagenum);
+
+    /* our READONLY copy *has* to have the current data, no
+       copy necessary */
+    /* make READONLY pages in other segments NO_ACCESS */
+    for (int i = 1; i < NB_SEGMENTS; i++) {
+        if (i == my_segnum)
+            continue;
+
+        if (get_page_status_in(i, pagenum) == PAGE_READONLY)
+            page_mark_inaccessible(i, pagenum);
+    }
+
+}
+
 long ro_to_acc = 0;
-static void handle_segfault_in_page(uintptr_t pagenum)
+static void handle_segfault_in_page(uintptr_t pagenum, bool is_write)
 {
     /* assumes page 'pagenum' is ACCESS_NONE, privatizes it,
        and validates to newest revision */
@@ -78,22 +96,13 @@
     assert(page_status == PAGE_NO_ACCESS
            || page_status == PAGE_READONLY);
 
+
+    //is_write=false;
+    if (page_status == PAGE_READONLY || is_write) {
+        dprintf(("SHORTCUT\n"));
+        readonly_to_accessible(my_segnum, pagenum);
+    }
     if (page_status == PAGE_READONLY) {
-        /* make our page write-ready */
-        page_mark_accessible(my_segnum, pagenum);
-
-        dprintf((" > found READONLY, make others NO_ACCESS\n"));
-        /* our READONLY copy *has* to have the current data, no
-           copy necessary */
-        /* make READONLY pages in other segments NO_ACCESS */
-        for (i = 1; i < NB_SEGMENTS; i++) {
-            if (i == my_segnum)
-                continue;
-
-            if (get_page_status_in(i, pagenum) == PAGE_READONLY)
-                page_mark_inaccessible(i, pagenum);
-        }
-
         ro_to_acc++;
 
         release_all_privatization_locks();
@@ -155,7 +164,8 @@
     }
 
     /* make our page write-ready */
-    page_mark_accessible(my_segnum, pagenum);
+    if (!is_write) // is_write -> already marked accessible above
+        page_mark_accessible(my_segnum, pagenum);
 
     /* account for this page now: XXX */
     /* increment_total_allocated(4096); */
@@ -230,8 +240,11 @@
         abort();
     }
 
+    // http://stackoverflow.com/questions/17671869/how-to-identify-read-or-write-operations-of-page-fault-when-using-sigaction-hand
+    bool is_write = ((ucontext_t*)context)->uc_mcontext.gregs[REG_ERR] & 0x2;
+
     DEBUG_EXPECT_SEGFAULT(false);
-    handle_segfault_in_page(pagenum);
+    handle_segfault_in_page(pagenum, is_write);
     DEBUG_EXPECT_SEGFAULT(true);
 
     errno = saved_errno;
diff --git a/c8/stm/signal_handler.h b/c8/stm/signal_handler.h
--- a/c8/stm/signal_handler.h
+++ b/c8/stm/signal_handler.h
@@ -2,7 +2,7 @@
 static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum,
                                       bool only_if_not_modified);
 
-static void handle_segfault_in_page(uintptr_t pagenum);
+static void handle_segfault_in_page(uintptr_t pagenum, bool is_write);
 
 
 static void setup_signal_handler(void);

From pypy.commits at gmail.com  Mon Feb 27 11:47:26 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 27 Feb 2017 08:47:26 -0800 (PST)
Subject: [pypy-commit] pypy default: (plan_rich, mjacob,
 ronan) document a difference on pypy3
Message-ID: <58b4581e.8d63190a.95ae7.afc0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r90408:b0cf041ff37c
Date: 2017-02-27 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/b0cf041ff37c/

Log:	(plan_rich, mjacob, ronan) document a difference on pypy3

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -483,6 +483,11 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
+* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+  CPython allows that for module subtypes, but not for e.g. ``int``
+  or ``float`` subtypes. Currently PyPy does not support the
+  ``__class__`` attribute assignment for any non heaptype subtype.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/

From pypy.commits at gmail.com  Mon Feb 27 11:51:25 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 27 Feb 2017 08:51:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: (plan_rich,
 mjacob) Mark this as an implementation detail (documented in
 b0cf041ff37c315390a54bf119dae8cc0b270a44).
Message-ID: <58b4590d.84d4190a.1a321.1bd4@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r90409:3371879bfdd7
Date: 2017-02-27 17:51 +0100
http://bitbucket.org/pypy/pypy/changeset/3371879bfdd7/

Log:	(plan_rich, mjacob) Mark this as an implementation detail
	(documented in b0cf041ff37c315390a54bf119dae8cc0b270a44).

diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -1022,20 +1022,21 @@
         self.assertEqual(x.__dict__, {'foo': 1})
 
     def test_object_class_assignment_between_heaptypes_and_nonheaptypes(self):
-        class SubType(types.ModuleType):
-            a = 1
-
-        m = types.ModuleType("m")
-        self.assertTrue(m.__class__ is types.ModuleType)
-        self.assertFalse(hasattr(m, "a"))
-
-        m.__class__ = SubType
-        self.assertTrue(m.__class__ is SubType)
-        self.assertTrue(hasattr(m, "a"))
-
-        m.__class__ = types.ModuleType
-        self.assertTrue(m.__class__ is types.ModuleType)
-        self.assertFalse(hasattr(m, "a"))
+        if support.check_impl_detail(pypy=False):
+            class SubType(types.ModuleType):
+                a = 1
+
+            m = types.ModuleType("m")
+            self.assertTrue(m.__class__ is types.ModuleType)
+            self.assertFalse(hasattr(m, "a"))
+
+            m.__class__ = SubType
+            self.assertTrue(m.__class__ is SubType)
+            self.assertTrue(hasattr(m, "a"))
+
+            m.__class__ = types.ModuleType
+            self.assertTrue(m.__class__ is types.ModuleType)
+            self.assertFalse(hasattr(m, "a"))
 
         # Make sure that builtin immutable objects don't support __class__
         # assignment, because the object instances may be interned.

From pypy.commits at gmail.com  Tue Feb 28 03:10:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 00:10:23 -0800 (PST)
Subject: [pypy-commit] cffi default: Document more explicitly the difference
 between CompiledFFI and FFI.
Message-ID: <58b5306f.0be3190a.83fb9.0e82@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2895:46f49821e351
Date: 2017-02-28 09:10 +0100
http://bitbucket.org/cffi/cffi/changeset/46f49821e351/

Log:	Document more explicitly the difference between CompiledFFI and FFI.

diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -8,6 +8,14 @@
 FFI Interface
 -------------
 
+*This page documents the runtime interface of the two types "FFI" and
+"CompiledFFI".  These two types are very similar to each other.  You get
+a CompiledFFI object if you import an out-of-line module.  You get a FFI
+object from explicitly writing cffi.FFI().  Unlike CompiledFFI, the type
+FFI has also got additional methods documented on the* `next page`__.
+
+.. __: cdef.html
+
 
 ffi.NULL
 ++++++++

From pypy.commits at gmail.com  Tue Feb 28 04:59:02 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Feb 2017 01:59:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Merged in
 robert-zaremba/pypy/py3.5-fix-globals (pull request #521)
Message-ID: <58b549e6.04162e0a.78a14.152e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90412:0436a099e10a
Date: 2017-02-28 09:58 +0000
http://bitbucket.org/pypy/pypy/changeset/0436a099e10a/

Log:	Merged in robert-zaremba/pypy/py3.5-fix-globals (pull request #521)

	(stevie, robert-zaremba)

	Approved-by: Richard Plangger

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -8,7 +8,7 @@
 # These are for internal use only:
 SYM_BLANK = 0
 SYM_GLOBAL = 1
-SYM_ASSIGNED = 2 # Or deleted actually.
+SYM_ASSIGNED = 2  # (DEF_LOCAL in CPython3). Or deleted actually.
 SYM_PARAM = 2 << 1
 SYM_NONLOCAL = 2 << 2
 SYM_USED = 2 << 3
@@ -123,12 +123,6 @@
     def _finalize_name(self, name, flags, local, bound, free, globs):
         """Decide on the scope of a name."""
         if flags & SYM_GLOBAL:
-            if flags & SYM_PARAM:
-                err = "name '%s' is parameter and global" % (name,)
-                raise SyntaxError(err, self.lineno, self.col_offset)
-            if flags & SYM_NONLOCAL:
-                err = "name '%s' is nonlocal and global" % (name,)
-                raise SyntaxError(err, self.lineno, self.col_offset)
             self.symbols[name] = SCOPE_GLOBAL_EXPLICIT
             globs[name] = None
             if bound:
@@ -137,12 +131,6 @@
                 except KeyError:
                     pass
         elif flags & SYM_NONLOCAL:
-            if flags & SYM_PARAM:
-                err = "name '%s' is parameter and nonlocal" % (name,)
-                raise SyntaxError(err, self.lineno, self.col_offset)
-            if bound is None:
-                err = "nonlocal declaration not allowed at module level"
-                raise SyntaxError(err, self.lineno, self.col_offset)
             if name not in bound:
                 err = "no binding for nonlocal '%s' found" % (name,)
                 raise SyntaxError(err, self.lineno, self.col_offset)
@@ -293,7 +281,8 @@
 
     def _pass_on_bindings(self, local, bound, globs, new_bound, new_globs):
         new_bound.update(local)
-        Scope._pass_on_bindings(self, local, bound, globs, new_bound, new_globs)
+        Scope._pass_on_bindings(self, local, bound, globs, new_bound,
+                                new_globs)
 
     def _finalize_cells(self, free):
         for name, role in self.symbols.iteritems():
@@ -496,20 +485,38 @@
                        "implemented in PyPy")
                 raise SyntaxError(msg, glob.lineno, glob.col_offset,
                                   filename=self.compile_info.filename)
+            if old_role & SYM_PARAM:
+                msg = "name '%s' is parameter and global" % (name,)
+                raise SyntaxError(msg, glob.lineno, glob.col_offset)
+            if old_role & SYM_NONLOCAL:
+                msg = "name '%s' is nonlocal and global" % (name,)
+                raise SyntaxError(msg, glob.lineno, glob.col_offset)
+
             if old_role & (SYM_USED | SYM_ASSIGNED):
                 if old_role & SYM_ASSIGNED:
-                    msg = "name '%s' is assigned to before global declaration" \
+                    msg = "name '%s' is assigned to before global declaration"\
                         % (name,)
                 else:
                     msg = "name '%s' is used prior to global declaration" % \
                         (name,)
-                misc.syntax_warning(self.space, msg, self.compile_info.filename,
+                misc.syntax_warning(self.space, msg,
+                                    self.compile_info.filename,
                                     glob.lineno, glob.col_offset)
             self.note_symbol(name, SYM_GLOBAL)
 
     def visit_Nonlocal(self, nonl):
         for name in nonl.names:
             old_role = self.scope.lookup_role(name)
+            msg = ""
+            if old_role & SYM_GLOBAL:
+                msg = "name '%s' is nonlocal and global" % (name,)
+            if old_role & SYM_PARAM:
+                msg = "name '%s' is parameter and nonlocal" % (name,)
+            if type(self.scope) == ModuleScope:
+                msg = "nonlocal declaration not allowed at module level"
+            if msg is not "":
+                raise SyntaxError(msg, nonl.lineno, nonl.col_offset)
+
             if (old_role & (SYM_USED | SYM_ASSIGNED) and not
                     (name == '__class__' and
                      self.scope._hide_bound_from_nested_scopes)):
@@ -519,8 +526,10 @@
                 else:
                     msg = "name '%s' is used prior to nonlocal declaration" % \
                         (name,)
-                misc.syntax_warning(self.space, msg, self.compile_info.filename,
+                misc.syntax_warning(self.space, msg,
+                                    self.compile_info.filename,
                                     nonl.lineno, nonl.col_offset)
+
             self.note_symbol(name, SYM_NONLOCAL)
 
     def visit_Lambda(self, lamb):
@@ -589,7 +598,7 @@
 
     def visit_arguments(self, arguments):
         scope = self.scope
-        assert isinstance(scope, FunctionScope) # Annotator hint.
+        assert isinstance(scope, FunctionScope)  # Annotator hint.
         if arguments.args:
             self._handle_params(arguments.args, True)
         if arguments.kwonlyargs:
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -283,7 +283,6 @@
     def test_global(self):
         scp = self.func_scope("def f():\n   global x\n   x = 4")
         assert scp.lookup("x") == symtable.SCOPE_GLOBAL_EXPLICIT
-        input = "def f(x):\n   global x"
         scp = self.func_scope("""def f():
     y = 3
     def x():
@@ -295,18 +294,38 @@
         xscp, zscp = scp.children
         assert xscp.lookup("y") == symtable.SCOPE_GLOBAL_EXPLICIT
         assert zscp.lookup("y") == symtable.SCOPE_FREE
-        exc = py.test.raises(SyntaxError, self.func_scope, input).value
+
+        src = "def f(x):\n   global x"
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.lineno == 2
         assert exc.msg == "name 'x' is parameter and global"
 
+    def test_global_nested(self):
+        src = """
+def f(x):
+    def g(x):
+        global x"""
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.lineno == 4
+        assert exc.msg == "name 'x' is parameter and global"
+
+        scp = self.func_scope("""
+def f(x):
+    def g():
+        global x""")
+        g = scp.children[0]
+        assert g.name == 'g'
+        x = g.lookup_role('x')
+        assert x == symtable.SYM_GLOBAL
+
     def test_nonlocal(self):
-        src = str(py.code.Source("""
-                     def f():
-                         nonlocal x
-                         global x
-                 """))
+        src = """
+x = 1
+def f():
+    nonlocal x"""
         exc = py.test.raises(SyntaxError, self.func_scope, src).value
-        assert exc.msg == "name 'x' is nonlocal and global"
-        #
+        assert exc.msg == "no binding for nonlocal 'x' found"
+
         src = str(py.code.Source("""
                      def f(x):
                          nonlocal x
@@ -324,6 +343,58 @@
         src = "nonlocal x"
         exc = py.test.raises(SyntaxError, self.func_scope, src).value
         assert exc.msg == "nonlocal declaration not allowed at module level"
+        assert exc.lineno == 1
+
+        src = "x = 2\nnonlocal x"
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "nonlocal declaration not allowed at module level"
+        assert exc.lineno == 2
+
+    def test_nonlocal_and_global(self):
+        """This test differs from CPython3 behaviour. CPython points to the
+        first occurance of the global/local expression. PyPy will point to the
+        last expression which makes the problem."""
+        src = """
+def f():
+    nonlocal x
+    global x"""
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "name 'x' is nonlocal and global"
+        assert exc.lineno == 4
+
+        src = """
+def f():
+    global x
+    nonlocal x """
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "name 'x' is nonlocal and global"
+        assert exc.lineno == 4
+
+    def test_nonlocal_nested(self):
+        scp = self.func_scope("""
+def f(x):
+    def g():
+        nonlocal x""")
+        g = scp.children[0]
+        x = g.lookup_role('x')
+        assert x == symtable.SYM_NONLOCAL
+
+        scp = self.func_scope("""
+def f():
+    def g():
+        nonlocal x
+    x = 1""")
+        g = scp.children[0]
+        x = g.lookup_role('x')
+        assert x == symtable.SYM_NONLOCAL
+
+        src = """
+def f(x):
+    def g(x):
+        nonlocal x"""
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "name 'x' is parameter and nonlocal"
+        assert exc.lineno == 4
 
     def test_optimization(self):
         assert not self.mod_scope("").can_be_optimized

From pypy.commits at gmail.com  Tue Feb 28 04:59:10 2017
From: pypy.commits at gmail.com (robert-zaremba)
Date: Tue, 28 Feb 2017 01:59:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fix-globals: (stevie, robert-zaremba)
Message-ID: <58b549ee.8d63190a.e021b.1414@mx.google.com>

Author: Robert Zaremba <robert.zaremba at scale-it.pl>
Branch: py3.5-fix-globals
Changeset: r90410:baf2561c870c
Date: 2017-02-27 13:36 +0100
http://bitbucket.org/pypy/pypy/changeset/baf2561c870c/

Log:	(stevie, robert-zaremba)

	Test and fix global variable reporting error.

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -8,7 +8,7 @@
 # These are for internal use only:
 SYM_BLANK = 0
 SYM_GLOBAL = 1
-SYM_ASSIGNED = 2 # Or deleted actually.
+SYM_ASSIGNED = 2  # Or deleted actually.
 SYM_PARAM = 2 << 1
 SYM_NONLOCAL = 2 << 2
 SYM_USED = 2 << 3
@@ -123,9 +123,6 @@
     def _finalize_name(self, name, flags, local, bound, free, globs):
         """Decide on the scope of a name."""
         if flags & SYM_GLOBAL:
-            if flags & SYM_PARAM:
-                err = "name '%s' is parameter and global" % (name,)
-                raise SyntaxError(err, self.lineno, self.col_offset)
             if flags & SYM_NONLOCAL:
                 err = "name '%s' is nonlocal and global" % (name,)
                 raise SyntaxError(err, self.lineno, self.col_offset)
@@ -293,7 +290,8 @@
 
     def _pass_on_bindings(self, local, bound, globs, new_bound, new_globs):
         new_bound.update(local)
-        Scope._pass_on_bindings(self, local, bound, globs, new_bound, new_globs)
+        Scope._pass_on_bindings(self, local, bound, globs, new_bound,
+                                new_globs)
 
     def _finalize_cells(self, free):
         for name, role in self.symbols.iteritems():
@@ -496,14 +494,19 @@
                        "implemented in PyPy")
                 raise SyntaxError(msg, glob.lineno, glob.col_offset,
                                   filename=self.compile_info.filename)
+            if old_role & SYM_PARAM:
+                msg = "name '%s' is parameter and global" % (name,)
+                raise SyntaxError(msg, glob.lineno, glob.col_offset)
+
             if old_role & (SYM_USED | SYM_ASSIGNED):
                 if old_role & SYM_ASSIGNED:
-                    msg = "name '%s' is assigned to before global declaration" \
+                    msg = "name '%s' is assigned to before global declaration"\
                         % (name,)
                 else:
                     msg = "name '%s' is used prior to global declaration" % \
                         (name,)
-                misc.syntax_warning(self.space, msg, self.compile_info.filename,
+                misc.syntax_warning(self.space, msg,
+                                    self.compile_info.filename,
                                     glob.lineno, glob.col_offset)
             self.note_symbol(name, SYM_GLOBAL)
 
@@ -519,7 +522,8 @@
                 else:
                     msg = "name '%s' is used prior to nonlocal declaration" % \
                         (name,)
-                misc.syntax_warning(self.space, msg, self.compile_info.filename,
+                misc.syntax_warning(self.space, msg,
+                                    self.compile_info.filename,
                                     nonl.lineno, nonl.col_offset)
             self.note_symbol(name, SYM_NONLOCAL)
 
@@ -589,7 +593,7 @@
 
     def visit_arguments(self, arguments):
         scope = self.scope
-        assert isinstance(scope, FunctionScope) # Annotator hint.
+        assert isinstance(scope, FunctionScope)  # Annotator hint.
         if arguments.args:
             self._handle_params(arguments.args, True)
         if arguments.kwonlyargs:
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -283,7 +283,6 @@
     def test_global(self):
         scp = self.func_scope("def f():\n   global x\n   x = 4")
         assert scp.lookup("x") == symtable.SCOPE_GLOBAL_EXPLICIT
-        input = "def f(x):\n   global x"
         scp = self.func_scope("""def f():
     y = 3
     def x():
@@ -295,9 +294,30 @@
         xscp, zscp = scp.children
         assert xscp.lookup("y") == symtable.SCOPE_GLOBAL_EXPLICIT
         assert zscp.lookup("y") == symtable.SCOPE_FREE
-        exc = py.test.raises(SyntaxError, self.func_scope, input).value
+
+        code = "def f(x):\n   global x"
+        exc = py.test.raises(SyntaxError, self.func_scope, code).value
+        assert exc.lineno == 2
         assert exc.msg == "name 'x' is parameter and global"
 
+    def test_global_nested(self):
+        code = """
+def f(x):
+    def g(x):
+        global x"""
+        exc = py.test.raises(SyntaxError, self.func_scope, code).value
+        assert exc.lineno == 4
+        assert exc.msg == "name 'x' is parameter and global"
+
+        scp = self.func_scope("""
+def f(x):
+    def g():
+        global x""")
+        g = scp.children[0]
+        assert g.name == 'g'
+        x = g.lookup_role('x')
+        assert x == symtable.SYM_GLOBAL
+
     def test_nonlocal(self):
         src = str(py.code.Source("""
                      def f():

From pypy.commits at gmail.com  Tue Feb 28 04:59:12 2017
From: pypy.commits at gmail.com (robert-zaremba)
Date: Tue, 28 Feb 2017 01:59:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fix-globals: (stevie,
 robert-zaremba) Fix symtable for nonlocal
Message-ID: <58b549f0.84472e0a.34678.1438@mx.google.com>

Author: Robert Zaremba <robert.zaremba at scale-it.pl>
Branch: py3.5-fix-globals
Changeset: r90411:802f8b2e090c
Date: 2017-02-27 17:56 +0100
http://bitbucket.org/pypy/pypy/changeset/802f8b2e090c/

Log:	(stevie, robert-zaremba) Fix symtable for nonlocal

	Added tests and updated functionality for symtable construction for
	nonlocal declarations.

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -8,7 +8,7 @@
 # These are for internal use only:
 SYM_BLANK = 0
 SYM_GLOBAL = 1
-SYM_ASSIGNED = 2  # Or deleted actually.
+SYM_ASSIGNED = 2  # (DEF_LOCAL in CPython3). Or deleted actually.
 SYM_PARAM = 2 << 1
 SYM_NONLOCAL = 2 << 2
 SYM_USED = 2 << 3
@@ -123,9 +123,6 @@
     def _finalize_name(self, name, flags, local, bound, free, globs):
         """Decide on the scope of a name."""
         if flags & SYM_GLOBAL:
-            if flags & SYM_NONLOCAL:
-                err = "name '%s' is nonlocal and global" % (name,)
-                raise SyntaxError(err, self.lineno, self.col_offset)
             self.symbols[name] = SCOPE_GLOBAL_EXPLICIT
             globs[name] = None
             if bound:
@@ -134,12 +131,6 @@
                 except KeyError:
                     pass
         elif flags & SYM_NONLOCAL:
-            if flags & SYM_PARAM:
-                err = "name '%s' is parameter and nonlocal" % (name,)
-                raise SyntaxError(err, self.lineno, self.col_offset)
-            if bound is None:
-                err = "nonlocal declaration not allowed at module level"
-                raise SyntaxError(err, self.lineno, self.col_offset)
             if name not in bound:
                 err = "no binding for nonlocal '%s' found" % (name,)
                 raise SyntaxError(err, self.lineno, self.col_offset)
@@ -497,6 +488,9 @@
             if old_role & SYM_PARAM:
                 msg = "name '%s' is parameter and global" % (name,)
                 raise SyntaxError(msg, glob.lineno, glob.col_offset)
+            if old_role & SYM_NONLOCAL:
+                msg = "name '%s' is nonlocal and global" % (name,)
+                raise SyntaxError(msg, glob.lineno, glob.col_offset)
 
             if old_role & (SYM_USED | SYM_ASSIGNED):
                 if old_role & SYM_ASSIGNED:
@@ -513,6 +507,16 @@
     def visit_Nonlocal(self, nonl):
         for name in nonl.names:
             old_role = self.scope.lookup_role(name)
+            msg = ""
+            if old_role & SYM_GLOBAL:
+                msg = "name '%s' is nonlocal and global" % (name,)
+            if old_role & SYM_PARAM:
+                msg = "name '%s' is parameter and nonlocal" % (name,)
+            if type(self.scope) == ModuleScope:
+                msg = "nonlocal declaration not allowed at module level"
+            if msg is not "":
+                raise SyntaxError(msg, nonl.lineno, nonl.col_offset)
+
             if (old_role & (SYM_USED | SYM_ASSIGNED) and not
                     (name == '__class__' and
                      self.scope._hide_bound_from_nested_scopes)):
@@ -525,6 +529,7 @@
                 misc.syntax_warning(self.space, msg,
                                     self.compile_info.filename,
                                     nonl.lineno, nonl.col_offset)
+
             self.note_symbol(name, SYM_NONLOCAL)
 
     def visit_Lambda(self, lamb):
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -295,17 +295,17 @@
         assert xscp.lookup("y") == symtable.SCOPE_GLOBAL_EXPLICIT
         assert zscp.lookup("y") == symtable.SCOPE_FREE
 
-        code = "def f(x):\n   global x"
-        exc = py.test.raises(SyntaxError, self.func_scope, code).value
+        src = "def f(x):\n   global x"
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
         assert exc.lineno == 2
         assert exc.msg == "name 'x' is parameter and global"
 
     def test_global_nested(self):
-        code = """
+        src = """
 def f(x):
     def g(x):
         global x"""
-        exc = py.test.raises(SyntaxError, self.func_scope, code).value
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
         assert exc.lineno == 4
         assert exc.msg == "name 'x' is parameter and global"
 
@@ -319,14 +319,13 @@
         assert x == symtable.SYM_GLOBAL
 
     def test_nonlocal(self):
-        src = str(py.code.Source("""
-                     def f():
-                         nonlocal x
-                         global x
-                 """))
+        src = """
+x = 1
+def f():
+    nonlocal x"""
         exc = py.test.raises(SyntaxError, self.func_scope, src).value
-        assert exc.msg == "name 'x' is nonlocal and global"
-        #
+        assert exc.msg == "no binding for nonlocal 'x' found"
+
         src = str(py.code.Source("""
                      def f(x):
                          nonlocal x
@@ -344,6 +343,58 @@
         src = "nonlocal x"
         exc = py.test.raises(SyntaxError, self.func_scope, src).value
         assert exc.msg == "nonlocal declaration not allowed at module level"
+        assert exc.lineno == 1
+
+        src = "x = 2\nnonlocal x"
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "nonlocal declaration not allowed at module level"
+        assert exc.lineno == 2
+
+    def test_nonlocal_and_global(self):
+        """This test differs from CPython3 behaviour. CPython points to the
+        first occurance of the global/local expression. PyPy will point to the
+        last expression which makes the problem."""
+        src = """
+def f():
+    nonlocal x
+    global x"""
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "name 'x' is nonlocal and global"
+        assert exc.lineno == 4
+
+        src = """
+def f():
+    global x
+    nonlocal x """
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "name 'x' is nonlocal and global"
+        assert exc.lineno == 4
+
+    def test_nonlocal_nested(self):
+        scp = self.func_scope("""
+def f(x):
+    def g():
+        nonlocal x""")
+        g = scp.children[0]
+        x = g.lookup_role('x')
+        assert x == symtable.SYM_NONLOCAL
+
+        scp = self.func_scope("""
+def f():
+    def g():
+        nonlocal x
+    x = 1""")
+        g = scp.children[0]
+        x = g.lookup_role('x')
+        assert x == symtable.SYM_NONLOCAL
+
+        src = """
+def f(x):
+    def g(x):
+        nonlocal x"""
+        exc = py.test.raises(SyntaxError, self.func_scope, src).value
+        assert exc.msg == "name 'x' is parameter and nonlocal"
+        assert exc.lineno == 4
 
     def test_optimization(self):
         assert not self.mod_scope("").can_be_optimized

From pypy.commits at gmail.com  Tue Feb 28 05:40:18 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 28 Feb 2017 02:40:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge py3.5-text-utf8
Message-ID: <58b55392.01462e0a.765d5.16a4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90414:ee44cdf8b435
Date: 2017-02-28 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/ee44cdf8b435/

Log:	merge py3.5-text-utf8

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -67,7 +67,7 @@
 lib_pypy = os.path.join(os.path.dirname(__file__),
                         '..', '..', '..', 'lib_pypy')
 
- at unwrap_spec(modulename='text0', level=int)
+ at unwrap_spec(modulename='fsencode', level=int)
 def importhook(space, modulename, w_globals=None, w_locals=None, w_fromlist=None, level=0):
     # A minimal version, that can only import builtin and lib_pypy modules!
     assert w_locals is w_globals
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -5,6 +5,7 @@
 from pypy.interpreter.pycode import PyCode
 from pypy.module._io.interp_iobase import W_IOBase
 from pypy.interpreter.streamutil import wrap_streamerror
+from pypy.interpreter.error import OperationError
 
 
 def extension_suffixes(space):
@@ -72,7 +73,11 @@
     return None
 
 def is_builtin(space, w_name):
-    name = space.text0_w(w_name)
+    try:
+        name = space.text0_w(w_name)
+    except OperationError:
+        return space.newint(0)
+
     if name not in space.builtin_modules:
         return space.newint(0)
     if space.finditem(space.sys.get('modules'), w_name) is not None:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -255,6 +255,10 @@
     def test_import_keywords(self):
         __import__(name='sys', level=0)
 
+    def test_import_nonutf8_encodable(self):
+        exc = raises(ImportError, __import__, '\ud800')
+        assert exc.value.args[0].startswith("No module named ")
+
     def test_import_by_filename(self):
         import pkg.a
         filename = pkg.a.__file__
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -35,8 +35,7 @@
         w_uni = space.wrap(u'abcd')
         assert space.text_w(w_uni) == 'abcd'
         w_uni = space.wrap(unichr(0xd921) + unichr(0xdddd))
-        assert space.text_w(w_uni) == '\xed\xa4\xa1\xed\xb7\x9d'
-        #                             ^^^ and not the 4-bytes combined character
+        raises(UnicodeEncodeError, space.text_w, w_uni)
 
 
 class AppTestUnicodeStringStdOnly:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -8,7 +8,7 @@
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii,
-    unicode_encode_utf8sp)
+    unicode_encode_utf8_forbid_surrogates, SurrogateError)
 from rpython.rlib import jit
 
 from pypy.interpreter import unicodehelper
@@ -81,8 +81,16 @@
         return self._value
 
     def text_w(self, space):
-        identifier = jit.conditional_call_elidable(
-                            self._utf8, g_encode_utf8, self._value)
+        try:
+            identifier = jit.conditional_call_elidable(
+                                self._utf8, g_encode_utf8, self._value)
+        except SurrogateError as e:
+            raise OperationError(space.w_UnicodeEncodeError,
+                    space.newtuple([space.newtext('utf-8'),
+                                    self,
+                                    space.newint(e.index-1),
+                                    space.newint(e.index),
+                                    space.newtext("surrogates not allowed")]))
         if not jit.isconstant(self):
             self._utf8 = identifier
         return identifier
@@ -1257,7 +1265,7 @@
 @jit.elidable
 def g_encode_utf8(value):
     """This is a global function because of jit.conditional_call_value"""
-    return unicode_encode_utf8sp(value, len(value))
+    return unicode_encode_utf8_forbid_surrogates(value, len(value))
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -433,7 +433,9 @@
     return result.build()
 
 class SurrogateError(Exception):
-    pass
+    def __init__(self, char, index):
+        self.char = char
+        self.index = index
 
 def unicode_encode_utf8_forbid_surrogates(s, size):
     # Strict surrogate-forbidding utf-8 encoding.  Any surrogate character
@@ -454,7 +456,7 @@
             result.append(chr((0x80 | (ch & 0x3f))))
         elif ch < 0x10000:
             if 0xD800 <= ch <= 0xDFFF:
-                raise SurrogateError
+                raise SurrogateError(ch, pos)
             # Encode UCS2 Unicode ordinals
             result.append((chr((0xe0 | (ch >> 12)))))
             result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))

From pypy.commits at gmail.com  Tue Feb 28 05:40:21 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 28 Feb 2017 02:40:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: document branch
Message-ID: <58b55395.110f2e0a.5bc7f.167c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90415:51f8b20c486b
Date: 2017-02-28 11:38 +0100
http://bitbucket.org/pypy/pypy/changeset/51f8b20c486b/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -172,3 +172,7 @@
 .. branch: fix_bool_restype
 
 Fix for ``ctypes.c_bool``-returning ctypes functions
+
+.. branch: py3.5-text-utf8
+
+space.text_w now encodes to utf-8 not preserving surrogates.

From pypy.commits at gmail.com  Tue Feb 28 05:40:16 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 28 Feb 2017 02:40:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: (ronan,
 plan_rich) fix descriptor invocation inside the descr_call of
 W_TypeObject
Message-ID: <58b55390.95012e0a.9009e.1776@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90413:411407ffbeac
Date: 2017-02-27 18:55 +0100
http://bitbucket.org/pypy/pypy/changeset/411407ffbeac/

Log:	(ronan, plan_rich) fix descriptor invocation inside the descr_call
	of W_TypeObject

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -631,7 +631,7 @@
             if w_newdescr is None:    # see test_crash_mro_without_object_1
                 raise oefmt(space.w_TypeError, "cannot create '%N' instances",
                             self)
-            w_newfunc = space.get(w_newdescr, self)
+            w_newfunc = space.get(w_newdescr, space.w_None, w_type=self)
             if (space.config.objspace.std.newshortcut and
                 not we_are_jitted() and
                 isinstance(w_newtype, W_TypeObject)):
diff --git a/pypy/objspace/test/test_descriptor.py b/pypy/objspace/test/test_descriptor.py
--- a/pypy/objspace/test/test_descriptor.py
+++ b/pypy/objspace/test/test_descriptor.py
@@ -152,3 +152,13 @@
         assert hash(myfloat(-1.0)) == -2
         assert hash(myHashClass()) == -2
         assert hash(myHashClass3()) == hash(-10**100)
+
+    def test_descr_funny_new(self):
+        class C(object):
+            @classmethod
+            def __new__(*args):
+                return args
+
+        assert C.__new__(1,2) == (C, 1, 2)
+        assert C(1,2) == (C, C, 1, 2)
+

From pypy.commits at gmail.com  Tue Feb 28 05:40:23 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 28 Feb 2017 02:40:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <58b55397.95012e0a.9009e.1777@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r90416:38f6842e3229
Date: 2017-02-28 11:39 +0100
http://bitbucket.org/pypy/pypy/changeset/38f6842e3229/

Log:	merge heads

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -631,7 +631,7 @@
             if w_newdescr is None:    # see test_crash_mro_without_object_1
                 raise oefmt(space.w_TypeError, "cannot create '%N' instances",
                             self)
-            w_newfunc = space.get(w_newdescr, self)
+            w_newfunc = space.get(w_newdescr, space.w_None, w_type=self)
             if (space.config.objspace.std.newshortcut and
                 not we_are_jitted() and
                 isinstance(w_newtype, W_TypeObject)):
diff --git a/pypy/objspace/test/test_descriptor.py b/pypy/objspace/test/test_descriptor.py
--- a/pypy/objspace/test/test_descriptor.py
+++ b/pypy/objspace/test/test_descriptor.py
@@ -152,3 +152,13 @@
         assert hash(myfloat(-1.0)) == -2
         assert hash(myHashClass()) == -2
         assert hash(myHashClass3()) == hash(-10**100)
+
+    def test_descr_funny_new(self):
+        class C(object):
+            @classmethod
+            def __new__(*args):
+                return args
+
+        assert C.__new__(1,2) == (C, 1, 2)
+        assert C(1,2) == (C, C, 1, 2)
+

From pypy.commits at gmail.com  Tue Feb 28 05:43:44 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 28 Feb 2017 02:43:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-text-utf8: close branch
Message-ID: <58b55460.09512e0a.56eb0.16b6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-text-utf8
Changeset: r90417:136caa31d4f8
Date: 2017-02-28 11:43 +0100
http://bitbucket.org/pypy/pypy/changeset/136caa31d4f8/

Log:	close branch


From pypy.commits at gmail.com  Tue Feb 28 07:21:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 04:21:18 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe-2: (fijal,
 arigo) Starting to copy some part of the logic from nogil-unsafe
Message-ID: <58b56b3e.014b2e0a.2d1b4.1b80@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r90418:927663f424f4
Date: 2017-02-28 13:19 +0100
http://bitbucket.org/pypy/pypy/changeset/927663f424f4/

Log:	(fijal, arigo) Starting to copy some part of the logic from nogil-
	unsafe

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -71,7 +71,8 @@
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint
 from rpython.rlib.rarithmetic import LONG_BIT_SHIFT
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib import rthread
 from rpython.memory.gc.minimarkpage import out_of_memory
 
 #
@@ -188,6 +189,10 @@
 FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
 NURSARRAY = lltype.Array(llmemory.Address)
 
+NURSERY_FREE = rthread.ThreadLocalField(llmemory.Address, 'nursery_free')
+NURSERY_TOP  = rthread.ThreadLocalField(llmemory.Address, 'nursery_top')
+NEXT_NUBLOCK = rthread.ThreadLocalField(llmemory.Address, 'next_nublock')
+
 # ____________________________________________________________
 
 class IncrementalMiniMarkGC(MovingGCBase):
@@ -263,10 +268,19 @@
         "card_page_indices": 128,
 
         # Objects whose total size is at least 'large_object' bytes are
-        # allocated out of the nursery immediately, as old objects.  The
-        # minimal allocated size of the nursery is 2x the following
-        # number (by default, at least 132KB on 32-bit and 264KB on 64-bit).
-        "large_object": (16384+512)*WORD,
+        # allocated out of the nursery immediately, as old objects.
+        "large_object": 26000,
+
+        # Thread-local Block size: the nursery is divided into blocks of
+        # at most this size each, and allocations go on in a
+        # thread-local manner inside each block.  "large_object" must be
+        # significantly smaller, but at the same time the total nursery
+        # size must be many times bigger than "tl_block_size"; the minimum
+        # allocated nursery size is 2 times "tl_block_size".
+        # "cache_line_min" is used to round the actual thread-local
+        # blocks to a cache line, to avoid pointless cache conflicts.
+        "tl_block_size": 65536,
+        "cache_line_min": 256,
         }
 
     def __init__(self, config,
@@ -280,6 +294,8 @@
                  growth_rate_max=2.5,   # for tests
                  card_page_indices=0,
                  large_object=8*WORD,
+                 tl_block_size=9*WORD,
+                 cache_line_min=1*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
         "NOT_RPYTHON"
@@ -307,10 +323,12 @@
         # 'large_object' limit how big objects can be in the nursery, so
         # it gives a lower bound on the allowed size of the nursery.
         self.nonlarge_max = large_object - 1
+        self.tl_block_size = tl_block_size
+        self.cache_line_min = cache_line_min
         #
         self.nursery      = llmemory.NULL
-        self.nursery_free = llmemory.NULL
-        self.nursery_top  = llmemory.NULL
+        self.nublocks     = llmemory.NULL # <= linked list of threadlocal_base
+                                          # for all active nursery consumers
         self.debug_tiny_nursery = -1
         self.debug_rotating_nurseries = lltype.nullptr(NURSARRAY)
         self.extra_threshold = 0
@@ -431,7 +449,7 @@
         else:
             #
             defaultsize = self.nursery_size
-            minsize = 2 * (self.nonlarge_max + 1)
+            minsize = 2 * self.tl_block_size
             self.nursery_size = minsize
             self.allocate_nursery()
             #
@@ -507,6 +525,13 @@
             # Estimate this number conservatively
             bigobj = self.nonlarge_max + 1
             self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2)
+        #
+        # Round up
+        ll_assert((self.cache_line_min & (self.cache_line_min - 1)) == 0,
+                  "cache_line_min is not a power a two")
+        self.tl_block_size = ((self.tl_block_size + self.cache_line_min - 1)
+                              & ~(self.cache_line_min - 1))
+
 
     def _nursery_memory_size(self):
         extra = self.nonlarge_max + 1
@@ -526,10 +551,6 @@
         debug_start("gc-set-nursery-size")
         debug_print("nursery size:", self.nursery_size)
         self.nursery = self._alloc_nursery()
-        # the current position in the nursery:
-        self.nursery_free = self.nursery
-        # the end of the nursery:
-        self.nursery_top = self.nursery + self.nursery_size
         # initialize the threshold
         self.min_heap_size = max(self.min_heap_size, self.nursery_size *
                                               self.major_collection_threshold)
@@ -602,12 +623,27 @@
             #
             llarena.arena_protect(newnurs, self._nursery_memory_size(), False)
             self.nursery = newnurs
-            self.nursery_top = self.nursery + self.nursery_size
             debug_print("switching from nursery", oldnurs,
                         "to nursery", self.nursery,
                         "size", self.nursery_size)
             debug_stop("gc-debug")
 
+    get_nursery_free = staticmethod(NURSERY_FREE.getraw)
+    set_nursery_free = staticmethod(NURSERY_FREE.setraw)
+
+    get_nursery_top = staticmethod(NURSERY_TOP.getraw)
+    set_nursery_top = staticmethod(NURSERY_TOP.setraw)
+
+    get_next_nublock = staticmethod(NEXT_NUBLOCK.getraw)
+    set_next_nublock = staticmethod(NEXT_NUBLOCK.setraw)
+
+    @property
+    def nursery_top(self):
+        raise AssertionError, "fix caller"
+    @property
+    def nursery_free(self):
+        raise AssertionError, "fix caller"
+
 
     def malloc_fixedsize(self, typeid, size,
                                needs_finalizer=False,
@@ -645,10 +681,15 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.nursery_free
-            ll_assert(result != llmemory.NULL, "uninitialized nursery")
-            self.nursery_free = new_free = result + totalsize
-            if new_free > self.nursery_top:
+            result = self.get_nursery_free()
+            if not we_are_translated() and result == llmemory.NULL:
+                # can't do arithmetic from NULL when non-translated
+                grab_next_block = True
+            else:
+                new_free = result + totalsize
+                self.set_nursery_free(new_free)
+                grab_next_block = new_free > self.get_nursery_top()
+            if grab_next_block:
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -799,7 +840,7 @@
 
         minor_collection_count = 0
         while True:
-            self.nursery_free = llmemory.NULL      # debug: don't use me
+            self.set_nursery_free(llmemory.NULL)      # debug: don't use me
             # note: no "raise MemoryError" between here and the next time
             # we initialize nursery_free!
 
@@ -816,7 +857,7 @@
                 #     v     v    v  jump over this
                 # +---------+--------+--------+--------+-----------+ }
                 # | used    | pinned | empty  | pinned |  empty    | }- nursery
-                # +---------+--------+--------+--------+-----------+ }
+                # +---------+--------B--------B--------B-----------B }
                 #                       ^- try reserving totalsize in here next
                 #
                 # All pinned objects are represented by entries in
@@ -827,14 +868,12 @@
                 # totalsize) starts at the end of the pinned object and ends at
                 # nursery's end.
                 #
-                # find the size of the pinned object after nursery_top
-                size_gc_header = self.gcheaderbuilder.size_gc_header
-                pinned_obj_size = size_gc_header + self.get_size(
-                        self.nursery_top + size_gc_header)
+                # In the diagram above, self.nursery_barriers contains
+                # four addresses which match the four "B".
                 #
                 # update used nursery space to allocate objects
-                self.nursery_free = self.nursery_top + pinned_obj_size
-                self.nursery_top = self.nursery_barriers.popleft()
+                self.set_nursery_free(self.nursery_barriers.popleft())
+                self.set_nursery_top(self.nursery_barriers.popleft())
             else:
                 minor_collection_count += 1
                 if minor_collection_count == 1:
@@ -862,10 +901,9 @@
             # Tried to do something about nursery_free overflowing
             # nursery_top before this point. Try to reserve totalsize now.
             # If this succeeds break out of loop.
-            result = self.nursery_free
-            if self.nursery_free + totalsize <= self.nursery_top:
-                self.nursery_free = result + totalsize
-                ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow")
+            result = self.get_nursery_free()
+            if result + totalsize <= self.get_nursery_top():
+                self.set_nursery_free(result + totalsize)
                 break
             #
         #
@@ -1750,53 +1788,83 @@
         # pointer.
         size_gc_header = self.gcheaderbuilder.size_gc_header
         nursery_barriers = self.AddressDeque()
-        prev = self.nursery
-        self.surviving_pinned_objects.sort()
+        if self.surviving_pinned_objects.non_empty():
+            self.surviving_pinned_objects.sort()
+            next_pinned_object = self.surviving_pinned_objects.pop()
+        else:
+            next_pinned_object = llmemory.NULL
         ll_assert(
             self.pinned_objects_in_nursery == \
             self.surviving_pinned_objects.length(),
             "pinned_objects_in_nursery != surviving_pinned_objects.length()")
-        while self.surviving_pinned_objects.non_empty():
+
+        # The following loop divides the nursery into small blocks whose
+        # size is generally about 'self.tl_block_size', but skipping
+        # over any pinned object.  Depending on the position of pinned
+        # objects, it is possible that one or two of these blocks are
+        # unusable because they are too small, but it should not matter.
+        prev = self.nursery
+        full_end = self.nursery + self.nursery_size
+
+        while True:
+            # Round up 'prev' to a multiple of 'cache_line_min'
+            prev_num = llmemory.cast_adr_to_int(prev)
+            prev += (-prev_num) & (self.cache_line_min - 1)
             #
-            cur = self.surviving_pinned_objects.pop()
-            ll_assert(
-                cur >= prev, "pinned objects encountered in backwards order")
+            # Compute the next TL block limit as 'cur1' and 'cur2'.
+            # These two addresses are normally equal to each other,
+            # but if there is a pinned object, then 'cur1' is the
+            # start of the pinned object and 'cur2' the end.
             #
-            # clear the arena between the last pinned object (or arena start)
-            # and the pinned object
-            free_range_size = llarena.getfakearenaaddress(cur) - prev
+            if full_end - prev <= self.tl_block_size:
+                cur1 = full_end
+            else:
+                cur1 = prev + self.tl_block_size
+            #
+            if next_pinned_object and next_pinned_object <= cur1:
+                cur1 = next_pinned_object
+                if self.surviving_pinned_objects.non_empty():
+                    next_pinned_object = self.surviving_pinned_objects.pop()
+                else:
+                    next_pinned_object = llmemory.NULL
+                ll_assert(cur1 >= prev,
+                          "pinned objects encountered in backwards order")
+                # clean up object's flags
+                obj = cur1 + size_gc_header
+                self.header(obj).tid &= ~GCFLAG_VISITED
+                # set up 'cur1' and 'cur2'
+                cur1 = llarena.getfakearenaaddress(cur1)
+                cur2 = cur1 + (size_gc_header + self.get_size(obj))
+            else:
+                # no pinned object in this TL block.
+                cur2 = cur1
+            #
+            # clear this block in the arena
+            free_range_size = cur1 - prev
             if self.gc_nursery_debug:
                 llarena.arena_reset(prev, free_range_size, 3)
             else:
                 llarena.arena_reset(prev, free_range_size, 0)
             #
-            # clean up object's flags
-            obj = cur + size_gc_header
-            self.header(obj).tid &= ~GCFLAG_VISITED
+            # create a new nursery barrier for the pinned object
+            nursery_barriers.append(cur1)    # pinned object
+            if cur1 == full_end:
+                break
+            nursery_barriers.append(cur2)    # end of pinned object
             #
-            # create a new nursery barrier for the pinned object
-            nursery_barriers.append(cur)
-            #
-            # update 'prev' to the end of the 'cur' object
-            prev = prev + free_range_size + \
-                (size_gc_header + self.get_size(obj))
+            # update 'prev' for the next iteration
+            prev = cur2
         #
-        # reset everything after the last pinned object till the end of the arena
+        ll_assert(not next_pinned_object, "bad pinned object location")
         if self.gc_nursery_debug:
-            llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 3)
             if not nursery_barriers.non_empty():   # no pinned objects
                 self.debug_rotate_nursery()
-        else:
-            llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 0)
-        #
-        # always add the end of the nursery to the list
-        nursery_barriers.append(self.nursery + self.nursery_size)
         #
         self.nursery_barriers = nursery_barriers
         self.surviving_pinned_objects.delete()
         #
-        self.nursery_free = self.nursery
-        self.nursery_top = self.nursery_barriers.popleft()
+        self.set_nursery_free(self.nursery)
+        self.set_nursery_top(self.nursery_barriers.popleft())
         #
         # clear GCFLAG_PINNED_OBJECT_PARENT_KNOWN from all parents in the list.
         self.old_objects_pointing_to_pinned.foreach(

From pypy.commits at gmail.com  Tue Feb 28 07:21:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 04:21:20 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe-2: (fijal, arigo) add the test
Message-ID: <58b56b40.85d5190a.684df.1b45@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r90419:ef99e1203aa4
Date: 2017-02-28 13:20 +0100
http://bitbucket.org/pypy/pypy/changeset/ef99e1203aa4/

Log:	(fijal, arigo) add the test

diff --git a/rpython/memory/gc/test/test_thread.py b/rpython/memory/gc/test/test_thread.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/test_thread.py
@@ -0,0 +1,37 @@
+import pytest
+from rpython.rtyper.lltypesystem import lltype
+from rpython.memory.gc import incminimark
+from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S
+
+
+class LocalsForTests:
+    pass
+
+
+class ThreadSwitcher:
+    all_threadlocals = [incminimark.NURSERY_FREE,
+                        incminimark.NURSERY_TOP,
+                        incminimark.NEXT_NUBLOCK,
+                        ]
+
+    def __init__(self):
+        self.cache = {}
+        self.switch_thread(0)
+
+    def switch_thread(self, thread_num):
+        for tl in self.all_threadlocals:
+            tl.local = self.cache.setdefault((tl, thread_num), LocalsForTests())
+
+
+class TestMultithreaded(BaseDirectGCTest):
+    from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
+
+    def test_malloc_fixedsize(self):
+        ts = ThreadSwitcher()
+        for i in range(1000):
+            p = self.malloc(S)
+            p.x = 42
+            assert p.prev == lltype.nullptr(S)
+            assert p.next == lltype.nullptr(S)
+            ts.switch_thread(i & 1)
+        #...

From pypy.commits at gmail.com  Tue Feb 28 07:54:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 04:54:10 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe-2: (fijal, arigo) test and fix
Message-ID: <58b572f2.1427190a.c65ae.1d57@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r90420:56c4cf2a149a
Date: 2017-02-28 13:53 +0100
http://bitbucket.org/pypy/pypy/changeset/56c4cf2a149a/

Log:	(fijal, arigo) test and fix

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1653,12 +1653,25 @@
     # ----------
     # Nursery collection
 
+    def zero_nursery_pointers_in_all_threads(self):
+        pass    # patched in test_thread
+
     def _minor_collection(self):
         """Perform a minor collection: find the objects from the nursery
         that remain alive and move them out."""
         #
         debug_start("gc-minor")
         #
+        # First, write NULLs in all thread's nursery_free and
+        # nursery_top
+        if we_are_translated():
+            def zero_nursery_pointers(arg, tl):
+                (tl + NURSERY_FREE.offset).address[0] = llmemory.NULL
+                (tl + NURSERY_TOP.offset).address[0] = llmemory.NULL
+            rthread.enum_all_threadlocals(zero_nursery_pointers, None)
+        else:
+            self.zero_nursery_pointers_in_all_threads()
+        #
         # All nursery barriers are invalid from this point on.  They
         # are evaluated anew as part of the minor collection.
         self.nursery_barriers.delete()
diff --git a/rpython/memory/gc/test/test_thread.py b/rpython/memory/gc/test/test_thread.py
--- a/rpython/memory/gc/test/test_thread.py
+++ b/rpython/memory/gc/test/test_thread.py
@@ -1,5 +1,5 @@
 import pytest
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.memory.gc import incminimark
 from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S
 
@@ -14,9 +14,15 @@
                         incminimark.NEXT_NUBLOCK,
                         ]
 
-    def __init__(self):
+    def __init__(self, gc):
         self.cache = {}
         self.switch_thread(0)
+        gc.zero_nursery_pointers_in_all_threads = self._zero_pointers
+
+    def _zero_pointers(self):
+        for (tl, thread_num), local in self.cache.items():
+            if tl in (incminimark.NURSERY_FREE, incminimark.NURSERY_TOP):
+                local.rawvalue = llmemory.NULL
 
     def switch_thread(self, thread_num):
         for tl in self.all_threadlocals:
@@ -27,11 +33,19 @@
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
 
     def test_malloc_fixedsize(self):
-        ts = ThreadSwitcher()
+        ts = ThreadSwitcher(self.gc)
         for i in range(1000):
             p = self.malloc(S)
             p.x = 42
             assert p.prev == lltype.nullptr(S)
             assert p.next == lltype.nullptr(S)
             ts.switch_thread(i & 1)
-        #...
+
+    def test_malloc_fixedsize_2(self):
+        ts = ThreadSwitcher(self.gc)
+        for i in range(1000):
+            p = self.malloc(S)
+            p.x = 42
+            assert p.prev == lltype.nullptr(S)
+            assert p.next == lltype.nullptr(S)
+            ts.switch_thread(0 if i % 5 < 2 else 1)
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -469,3 +469,15 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+
+
+ at specialize.arg(0)
+def enum_all_threadlocals(callback, arg):
+    p = llmemory.NULL
+    llop.threadlocalref_acquire(lltype.Void)
+    while True:
+        p = llop.threadlocalref_enum(llmemory.Address, p)
+        if not p:
+            break
+        callback(arg, p)
+    llop.threadlocalref_release(lltype.Void)

From pypy.commits at gmail.com  Tue Feb 28 09:00:56 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 28 Feb 2017 06:00:56 -0800 (PST)
Subject: [pypy-commit] pypy default: test showing a corner case of the
 interaction between finalizers and heap
Message-ID: <58b58298.18152e0a.8fd3c.2309@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r90421:846ed5d2ac16
Date: 2017-02-28 15:00 +0100
http://bitbucket.org/pypy/pypy/changeset/846ed5d2ac16/

Log:	test showing a corner case of the interaction between finalizers and
	heap caching in the JIT

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4636,3 +4636,28 @@
             return i
 
         self.meta_interp(f, [10])
+
+    @py.test.skip("loops!")
+    def test_finalizer_bug(self):
+        from rpython.rlib import rgc
+        driver = JitDriver(greens=[], reds=[])
+        class Fin(object):
+            @rgc.must_be_light_finalizer
+            def __del__(self):
+                holder[0].field = 7
+        class Un(object):
+            def __init__(self):
+                self.field = 0
+        holder = [Un()]
+
+        def f():
+            while True:
+                driver.jit_merge_point()
+                holder[0].field = 0
+                Fin()
+                if holder[0].field:
+                    break
+            return holder[0].field
+
+        f() # finishes
+        self.meta_interp(f, [])

From pypy.commits at gmail.com  Tue Feb 28 09:32:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 06:32:22 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe-2: progress
Message-ID: <58b589f6.4395190a.a6c86.2491@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r90422:643158ba7b39
Date: 2017-02-28 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/643158ba7b39/

Log:	progress

diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -104,7 +104,7 @@
         self.rootstackhook(collect_stack_root,
                            gcdata.root_stack_base, gcdata.root_stack_top)
 
-    def need_thread_support(self, gctransformer, getfn):
+    def need_thread_support_WITH_GIL(self, gctransformer, getfn):
         from rpython.rlib import rthread    # xxx fish
         gcdata = self.gcdata
         # the interfacing between the threads and the GC is done via
@@ -218,6 +218,49 @@
                                            annmodel.s_None,
                                            minimal_transform=False)
 
+    def need_thread_support(self, gctransformer, getfn):  # NO GIL VERSION
+        from rpython.rlib import rthread
+        gcdata = self.gcdata
+        # the interfacing between the threads and the GC is done via
+        # two completely ad-hoc operations at the moment:
+        # gc_thread_run and gc_thread_die.  See docstrings below.
+
+        tl_shadowstack = rthread.ThreadLocalField(llmemory.Address,
+                                                  'shadowstack')
+
+        def thread_setup():
+            allocate_shadow_stack()
+
+        def thread_run():
+            # If it's the first time we see this thread, allocate
+            # a shadowstack.
+            if tl_shadowstack.get_or_make_raw() == llmemory.NULL:
+                allocate_shadow_stack()
+
+        def allocate_shadow_stack():
+            root_stack_depth = 163840
+            root_stack_size = sizeofaddr * root_stack_depth
+            ss = llmemory.raw_malloc(root_stack_size)
+            if not ss:
+                raise MemoryError
+            tl_shadowstack.setraw(ss)
+        allocate_shadow_stack._dont_inline_ = True
+
+        def thread_die():
+            """Called just before the final GIL release done by a dying
+            thread.  After a thread_die(), no more gc operation should
+            occur in this thread.
+            """
+            p = tl_shadowstack.get_or_make_raw()
+            tl_shadowstack.setraw(llmemory.NULL)
+            llmemory.raw_free(p)
+
+        self.thread_setup = thread_setup
+        self.thread_run_ptr = getfn(thread_run, [], annmodel.s_None,
+                                    inline=True, minimal_transform=False)
+        self.thread_die_ptr = getfn(thread_die, [], annmodel.s_None,
+                                    minimal_transform=False)
+
     def need_stacklet_support(self, gctransformer, getfn):
         from rpython.rlib import _stacklet_shadowstack
         _stacklet_shadowstack.complete_destrptr(gctransformer)
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -2,6 +2,7 @@
 #define __PYPY_THREAD_H
 #include "precommondefs.h"
 #include <assert.h>
+#include <stdlib.h>
 
 #define RPY_TIMEOUT_T long long
 
@@ -41,19 +42,20 @@
 # define RPY_FASTGIL_LOCKED(x)   (x != 0)
 #endif
 
-RPY_EXTERN long rpy_fastgil;
+//RPY_EXTERN long rpy_fastgil;
 
 static inline void _RPyGilAcquire(void) {
-    long old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1);
-    if (old_fastgil != 0)
-        RPyGilAcquireSlowPath(old_fastgil);
+//    long old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1);
+//    if (old_fastgil != 0)
+//        RPyGilAcquireSlowPath(old_fastgil);
 }
 static inline void _RPyGilRelease(void) {
-    assert(RPY_FASTGIL_LOCKED(rpy_fastgil));
-    pypy_lock_release(&rpy_fastgil);
+//    assert(RPY_FASTGIL_LOCKED(rpy_fastgil));
+//    pypy_lock_release(&rpy_fastgil);
 }
 static inline long *_RPyFetchFastGil(void) {
-    return &rpy_fastgil;
+    abort();
+//    return &rpy_fastgil;
 }
 
 #endif
diff --git a/rpython/translator/c/src/thread_gil.c b/rpython/translator/c/src/thread_gil.c
--- a/rpython/translator/c/src/thread_gil.c
+++ b/rpython/translator/c/src/thread_gil.c
@@ -58,13 +58,13 @@
 
 void RPyGilAllocate(void)
 {
-    if (rpy_waiting_threads < 0) {
-        assert(rpy_waiting_threads == -42);
-        rpy_init_mutexes();
+//    if (rpy_waiting_threads < 0) {
+//        assert(rpy_waiting_threads == -42);
+//        rpy_init_mutexes();
 #ifdef HAVE_PTHREAD_ATFORK
-        pthread_atfork(NULL, NULL, rpy_init_mutexes);
+//        pthread_atfork(NULL, NULL, rpy_init_mutexes);
 #endif
-    }
+//    }
 }
 
 static void check_and_save_old_fastgil(long old_fastgil)
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1427,6 +1427,46 @@
                 and result.count('a') == 1
                 and result.count('d') == 6)
 
+    def test_thread_and_gc_nogil(self):
+        import time, gc
+        from rpython.rlib import rthread, rposix
+
+        def bootstrap():
+            rthread.gc_thread_start()
+            os.write(1, "hi there\n")
+            rthread.gc_thread_die()
+
+        def new_thread():
+            ident = rthread.start_new_thread(bootstrap, ())
+            return ident
+
+        def entry_point(argv):
+            os.write(1, "hello world\n")
+            # start 5 new threads
+            ident1 = new_thread()
+            ident2 = new_thread()
+            ident3 = new_thread()
+            ident4 = new_thread()
+            ident5 = new_thread()
+            # wait for the 5 threads to finish
+            time.sleep(1)
+            gc.collect()
+            return 0
+
+        def runme(no__thread):
+            t, cbuilder = self.compile(entry_point, no__thread=no__thread)
+            data = cbuilder.cmdexec('')
+            assert data.splitlines() == ['hello world',
+                                         '1 ok',
+                                         '2 ok',
+                                         '3 ok',
+                                         '4 ok',
+                                         '5 ok']
+
+        if SUPPORT__THREAD:
+            runme(no__thread=False)
+        runme(no__thread=True)
+
 
 class TestShared(StandaloneTests):
 

From pypy.commits at gmail.com  Tue Feb 28 09:49:31 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Feb 2017 06:49:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Merged in
 robert-zaremba/pypy/py3.5-fix-decimal-module-name (pull request #522)
Message-ID: <58b58dfb.13542e0a.69d3d.22e9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r90425:b7745d301387
Date: 2017-02-28 14:48 +0000
http://bitbucket.org/pypy/pypy/changeset/b7745d301387/

Log:	Merged in robert-zaremba/pypy/py3.5-fix-decimal-module-name (pull
	request #522)

	Py3.5 fix decimal module name

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -7,6 +7,8 @@
 .project
 .pydevproject
 __pycache__
+.venv
+.cache
 
 syntax: regexp
 ^testresult$
diff --git a/extra_tests/support.py b/extra_tests/support.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/support.py
@@ -0,0 +1,98 @@
+import contextlib
+import importlib
+import sys
+import warnings
+
+
+ at contextlib.contextmanager
+def _ignore_deprecated_imports(ignore=True):
+    """Context manager to suppress package and module deprecation
+    warnings when importing them.
+
+    If ignore is False, this context manager has no effect.
+    """
+    if ignore:
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", ".+ (module|package)",
+                                    DeprecationWarning)
+            yield
+    else:
+        yield
+
+
+def _save_and_remove_module(name, orig_modules):
+    """Helper function to save and remove a module from sys.modules
+
+    Raise ImportError if the module can't be imported.
+    """
+    # try to import the module and raise an error if it can't be imported
+    if name not in sys.modules:
+        __import__(name)
+        del sys.modules[name]
+    for modname in list(sys.modules):
+        if modname == name or modname.startswith(name + '.'):
+            orig_modules[modname] = sys.modules[modname]
+            del sys.modules[modname]
+
+def _save_and_block_module(name, orig_modules):
+    """Helper function to save and block a module in sys.modules
+
+    Return True if the module was in sys.modules, False otherwise.
+    """
+    saved = True
+    try:
+        orig_modules[name] = sys.modules[name]
+    except KeyError:
+        saved = False
+    sys.modules[name] = None
+    return saved
+
+
+def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
+    """Import and return a module, deliberately bypassing sys.modules.
+
+    This function imports and returns a fresh copy of the named Python module
+    by removing the named module from sys.modules before doing the import.
+    Note that unlike reload, the original module is not affected by
+    this operation.
+
+    *fresh* is an iterable of additional module names that are also removed
+    from the sys.modules cache before doing the import.
+
+    *blocked* is an iterable of module names that are replaced with None
+    in the module cache during the import to ensure that attempts to import
+    them raise ImportError.
+
+    The named module and any modules named in the *fresh* and *blocked*
+    parameters are saved before starting the import and then reinserted into
+    sys.modules when the fresh import is complete.
+
+    Module and package deprecation messages are suppressed during this import
+    if *deprecated* is True.
+
+    This function will raise ImportError if the named module cannot be
+    imported.
+    """
+    # NOTE: test_heapq, test_json and test_warnings include extra sanity checks
+    # to make sure that this utility function is working as expected
+    with _ignore_deprecated_imports(deprecated):
+        # Keep track of modules saved for later restoration as well
+        # as those which just need a blocking entry removed
+        orig_modules = {}
+        names_to_remove = []
+        _save_and_remove_module(name, orig_modules)
+        try:
+            for fresh_name in fresh:
+                _save_and_remove_module(fresh_name, orig_modules)
+            for blocked_name in blocked:
+                if not _save_and_block_module(blocked_name, orig_modules):
+                    names_to_remove.append(blocked_name)
+            fresh_module = importlib.import_module(name)
+        except ImportError:
+            fresh_module = None
+        finally:
+            for orig_name, module in orig_modules.items():
+                sys.modules[orig_name] = module
+            for name_to_remove in names_to_remove:
+                del sys.modules[name_to_remove]
+        return fresh_module
diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_decimal.py
@@ -0,0 +1,59 @@
+import pickle
+import sys
+
+from support import import_fresh_module
+
+C = import_fresh_module('decimal', fresh=['_decimal'])
+P = import_fresh_module('decimal', blocked=['_decimal'])
+# import _decimal as C
+# import _pydecimal as P
+
+
+class TestPythonAPI:
+
+    def check_equal(self, val, proto):
+        d = C.Decimal(val)
+        p = pickle.dumps(d, proto)
+        assert d == pickle.loads(p)
+
+    def test_C(self):
+        sys.modules["decimal"] = C
+        import decimal
+        d = decimal.Decimal('1')
+        assert isinstance(d, C.Decimal)
+        assert isinstance(d, decimal.Decimal)
+        assert isinstance(d.as_tuple(), C.DecimalTuple)
+
+        assert d == C.Decimal('1')
+
+    def test_pickle(self):
+        v = '-3.123e81723'
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            sys.modules["decimal"] = C
+            self.check_equal('-3.141590000', proto)
+            self.check_equal(v, proto)
+
+            cd = C.Decimal(v)
+            pd = P.Decimal(v)
+            cdt = cd.as_tuple()
+            pdt = pd.as_tuple()
+            assert cdt.__module__ == pdt.__module__
+
+            p = pickle.dumps(cdt, proto)
+            r = pickle.loads(p)
+            assert isinstance(r, C.DecimalTuple)
+            assert cdt == r
+
+            sys.modules["decimal"] = C
+            p = pickle.dumps(cd, proto)
+            sys.modules["decimal"] = P
+            r = pickle.loads(p)
+            assert isinstance(r, P.Decimal)
+            assert r == pd
+
+            sys.modules["decimal"] = C
+            p = pickle.dumps(cdt, proto)
+            sys.modules["decimal"] = P
+            r = pickle.loads(p)
+            assert isinstance(r, P.DecimalTuple)
+            assert r == pdt
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -1,5 +1,9 @@
 # Implementation of the "decimal" module, based on libmpdec library.
 
+__xname__ = __name__    # sys.modules lookup (--without-threads)
+__name__ = 'decimal'    # For pickling
+
+
 import collections as _collections
 import math as _math
 import numbers as _numbers
@@ -23,15 +27,13 @@
 # Errors
 
 class DecimalException(ArithmeticError):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         pass
 
 class Clamped(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class InvalidOperation(DecimalException):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         if args:
             ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True)
@@ -39,41 +41,35 @@
         return _NaN
 
 class ConversionSyntax(InvalidOperation):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class DivisionByZero(DecimalException, ZeroDivisionError):
-    __module__ = 'decimal'
     def handle(self, context, sign, *args):
         return _SignedInfinity[sign]
 
 class DivisionImpossible(InvalidOperation):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class DivisionUndefined(InvalidOperation, ZeroDivisionError):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class Inexact(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class InvalidContext(InvalidOperation):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class Rounded(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class Subnormal(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class Overflow(Inexact, Rounded):
-    __module__ = 'decimal'
     def handle(self, context, sign, *args):
         if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN,
                                 ROUND_HALF_DOWN, ROUND_UP):
@@ -90,10 +86,10 @@
                              context.Emax-context.prec+1)
 
 class Underflow(Inexact, Rounded, Subnormal):
-    __module__ = 'decimal'
+    pass
 
 class FloatOperation(DecimalException, TypeError):
-    __module__ = 'decimal'
+    pass
 
 
 __version__ = "1.70"
@@ -107,7 +103,7 @@
 
 def getcontext():
     """Returns this thread's context.
-    
+
     If this thread does not yet have a context, returns
     a new context and sets this thread's context.
     New contexts are copies of DefaultContext.
@@ -173,8 +169,6 @@
 _DEC_MINALLOC = 4
 
 class Decimal(object):
-    __module__ = 'decimal'
-
     __slots__ = ('_mpd', '_data')
 
     def __new__(cls, value="0", context=None):
@@ -326,7 +320,7 @@
             builder.append(b'E')
             builder.append(str(exponent).encode())
 
-        return cls._from_bytes(b''.join(builder), context, exact=exact) 
+        return cls._from_bytes(b''.join(builder), context, exact=exact)
 
     @classmethod
     def from_float(cls, value):
@@ -481,7 +475,7 @@
             numerator = Decimal._from_int(other.numerator, context)
             if not _mpdec.mpd_isspecial(self._mpd):
                 # multiplied = self * other.denominator
-                # 
+                #
                 # Prevent Overflow in the following multiplication.
                 # The result of the multiplication is
                 # only used in mpd_qcmp, which can handle values that
@@ -542,7 +536,7 @@
         _mpdec.mpd_qset_ssize(p._mpd, self._PyHASH_MODULUS,
                               maxctx, status_ptr)
         ten = self._new_empty()
-        _mpdec.mpd_qset_ssize(ten._mpd, 10, 
+        _mpdec.mpd_qset_ssize(ten._mpd, 10,
                               maxctx, status_ptr)
         inv10_p = self._new_empty()
         _mpdec.mpd_qset_ssize(inv10_p._mpd, self._PyHASH_10INV,
@@ -755,7 +749,7 @@
     number_class = _make_unary_operation('number_class')
 
     to_eng_string = _make_unary_operation('to_eng_string')
-    
+
     def fma(self, other, third, context=None):
         context = _getcontext(context)
         return context.fma(self, other, third)
@@ -790,7 +784,7 @@
             result = int.from_bytes(s, 'little', signed=False)
         if _mpdec.mpd_isnegative(x._mpd) and not _mpdec.mpd_iszero(x._mpd):
             result = -result
-        return result        
+        return result
 
     def __int__(self):
         return self._to_int(_mpdec.MPD_ROUND_DOWN)
@@ -798,10 +792,10 @@
     __trunc__ = __int__
 
     def __floor__(self):
-        return self._to_int(_mpdec.MPD_ROUND_FLOOR)        
+        return self._to_int(_mpdec.MPD_ROUND_FLOOR)
 
     def __ceil__(self):
-        return self._to_int(_mpdec.MPD_ROUND_CEILING)        
+        return self._to_int(_mpdec.MPD_ROUND_CEILING)
 
     def to_integral(self, rounding=None, context=None):
         context = _getcontext(context)
@@ -817,7 +811,7 @@
         return result
 
     to_integral_value = to_integral
-        
+
     def to_integral_exact(self, rounding=None, context=None):
         context = _getcontext(context)
         workctx = context.copy()
@@ -886,7 +880,7 @@
         if _mpdec.mpd_isspecial(self._mpd):
             return 0
         return _mpdec.mpd_adjexp(self._mpd)
-    
+
     @property
     def real(self):
         return self
@@ -916,7 +910,7 @@
         fmt = specifier.encode('utf-8')
         context = getcontext()
 
-        replace_fillchar = False 
+        replace_fillchar = False
         if fmt and fmt[0] == 0:
             # NUL fill character: must be replaced with a valid UTF-8 char
             # before calling mpd_parse_fmt_str().
@@ -975,7 +969,7 @@
             result = result.replace(b'\xff', b'\0')
         return result.decode('utf-8')
 
-        
+
 # Register Decimal as a kind of Number (an abstract base class).
 # However, do not register it as Real (because Decimals are not
 # interoperable with floats).
@@ -988,7 +982,7 @@
 
 # Rounding
 _ROUNDINGS = {
-    'ROUND_DOWN': _mpdec.MPD_ROUND_DOWN, 
+    'ROUND_DOWN': _mpdec.MPD_ROUND_DOWN,
     'ROUND_HALF_UP': _mpdec.MPD_ROUND_HALF_UP,
     'ROUND_HALF_EVEN': _mpdec.MPD_ROUND_HALF_EVEN,
     'ROUND_CEILING': _mpdec.MPD_ROUND_CEILING,
@@ -1047,8 +1041,6 @@
     clamp -  If 1, change exponents if too high (Default 0)
     """
 
-    __module__ = 'decimal'
-
     __slots__ = ('_ctx', '_capitals')
 
     def __new__(cls, prec=None, rounding=None, Emin=None, Emax=None,
@@ -1068,7 +1060,7 @@
         ctx.round = _mpdec.MPD_ROUND_HALF_EVEN
         ctx.clamp = 0
         ctx.allcr = 1
-        
+
         self._capitals = 1
         return self
 
@@ -1291,7 +1283,7 @@
         if b is NotImplemented:
             return b, b
         return a, b
-        
+
     def _make_unary_method(name, mpd_func_name):
         mpd_func = getattr(_mpdec, mpd_func_name)
 
@@ -1570,7 +1562,7 @@
 
     def copy(self):
         return self._as_dict()
-        
+
     def __len__(self):
         return len(_SIGNALS)
 
@@ -1629,7 +1621,7 @@
     def __enter__(self):
         self.status_ptr = _ffi.new("uint32_t*")
         return self.context._ctx, self.status_ptr
-        
+
     def __exit__(self, *args):
         status = self.status_ptr[0]
         # May raise a DecimalException

From pypy.commits at gmail.com  Tue Feb 28 09:49:41 2017
From: pypy.commits at gmail.com (robert-zaremba)
Date: Tue, 28 Feb 2017 06:49:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fix-decimal-module-name: Added .venv
 (virtualenvironment) and .cache to .hgignore
Message-ID: <58b58e05.15502e0a.e4311.24b2@mx.google.com>

Author: Robert Zaremba <robert.zaremba at scale-it.pl>
Branch: py3.5-fix-decimal-module-name
Changeset: r90423:cc54d3bbf8a5
Date: 2017-02-28 15:28 +0100
http://bitbucket.org/pypy/pypy/changeset/cc54d3bbf8a5/

Log:	Added .venv (virtualenvironment) and .cache to .hgignore

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -7,6 +7,8 @@
 .project
 .pydevproject
 __pycache__
+.venv
+.cache
 
 syntax: regexp
 ^testresult$

From pypy.commits at gmail.com  Tue Feb 28 09:49:43 2017
From: pypy.commits at gmail.com (robert-zaremba)
Date: Tue, 28 Feb 2017 06:49:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-fix-decimal-module-name: (stevie,
 robert-zaremba) FIX: test_pickle
 (test.test_decimal.PyPythonAPItests)
Message-ID: <58b58e07.0d57190a.928b4.2481@mx.google.com>

Author: Robert Zaremba <robert.zaremba at scale-it.pl>
Branch: py3.5-fix-decimal-module-name
Changeset: r90424:793b49cacddd
Date: 2017-02-28 15:32 +0100
http://bitbucket.org/pypy/pypy/changeset/793b49cacddd/

Log:	(stevie, robert-zaremba) FIX: test_pickle
	(test.test_decimal.PyPythonAPItests)

	Fixes: http://buildbot.pypy.org/summary/longrepr?testname=unmodified
	&builder=pypy-c-jit-linux-x86-64&build=4406&mod=lib-
	python.3.test.test_decimal

	We removed the __module__ hack from the classes in
	lib_pypy/_decimal.py and added module __name__ pointing to the right
	module name.

diff --git a/extra_tests/support.py b/extra_tests/support.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/support.py
@@ -0,0 +1,98 @@
+import contextlib
+import importlib
+import sys
+import warnings
+
+
+ at contextlib.contextmanager
+def _ignore_deprecated_imports(ignore=True):
+    """Context manager to suppress package and module deprecation
+    warnings when importing them.
+
+    If ignore is False, this context manager has no effect.
+    """
+    if ignore:
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", ".+ (module|package)",
+                                    DeprecationWarning)
+            yield
+    else:
+        yield
+
+
+def _save_and_remove_module(name, orig_modules):
+    """Helper function to save and remove a module from sys.modules
+
+    Raise ImportError if the module can't be imported.
+    """
+    # try to import the module and raise an error if it can't be imported
+    if name not in sys.modules:
+        __import__(name)
+        del sys.modules[name]
+    for modname in list(sys.modules):
+        if modname == name or modname.startswith(name + '.'):
+            orig_modules[modname] = sys.modules[modname]
+            del sys.modules[modname]
+
+def _save_and_block_module(name, orig_modules):
+    """Helper function to save and block a module in sys.modules
+
+    Return True if the module was in sys.modules, False otherwise.
+    """
+    saved = True
+    try:
+        orig_modules[name] = sys.modules[name]
+    except KeyError:
+        saved = False
+    sys.modules[name] = None
+    return saved
+
+
+def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
+    """Import and return a module, deliberately bypassing sys.modules.
+
+    This function imports and returns a fresh copy of the named Python module
+    by removing the named module from sys.modules before doing the import.
+    Note that unlike reload, the original module is not affected by
+    this operation.
+
+    *fresh* is an iterable of additional module names that are also removed
+    from the sys.modules cache before doing the import.
+
+    *blocked* is an iterable of module names that are replaced with None
+    in the module cache during the import to ensure that attempts to import
+    them raise ImportError.
+
+    The named module and any modules named in the *fresh* and *blocked*
+    parameters are saved before starting the import and then reinserted into
+    sys.modules when the fresh import is complete.
+
+    Module and package deprecation messages are suppressed during this import
+    if *deprecated* is True.
+
+    This function will raise ImportError if the named module cannot be
+    imported.
+    """
+    # NOTE: test_heapq, test_json and test_warnings include extra sanity checks
+    # to make sure that this utility function is working as expected
+    with _ignore_deprecated_imports(deprecated):
+        # Keep track of modules saved for later restoration as well
+        # as those which just need a blocking entry removed
+        orig_modules = {}
+        names_to_remove = []
+        _save_and_remove_module(name, orig_modules)
+        try:
+            for fresh_name in fresh:
+                _save_and_remove_module(fresh_name, orig_modules)
+            for blocked_name in blocked:
+                if not _save_and_block_module(blocked_name, orig_modules):
+                    names_to_remove.append(blocked_name)
+            fresh_module = importlib.import_module(name)
+        except ImportError:
+            fresh_module = None
+        finally:
+            for orig_name, module in orig_modules.items():
+                sys.modules[orig_name] = module
+            for name_to_remove in names_to_remove:
+                del sys.modules[name_to_remove]
+        return fresh_module
diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_decimal.py
@@ -0,0 +1,59 @@
+import pickle
+import sys
+
+from support import import_fresh_module
+
+C = import_fresh_module('decimal', fresh=['_decimal'])
+P = import_fresh_module('decimal', blocked=['_decimal'])
+# import _decimal as C
+# import _pydecimal as P
+
+
+class TestPythonAPI:
+
+    def check_equal(self, val, proto):
+        d = C.Decimal(val)
+        p = pickle.dumps(d, proto)
+        assert d == pickle.loads(p)
+
+    def test_C(self):
+        sys.modules["decimal"] = C
+        import decimal
+        d = decimal.Decimal('1')
+        assert isinstance(d, C.Decimal)
+        assert isinstance(d, decimal.Decimal)
+        assert isinstance(d.as_tuple(), C.DecimalTuple)
+
+        assert d == C.Decimal('1')
+
+    def test_pickle(self):
+        v = '-3.123e81723'
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            sys.modules["decimal"] = C
+            self.check_equal('-3.141590000', proto)
+            self.check_equal(v, proto)
+
+            cd = C.Decimal(v)
+            pd = P.Decimal(v)
+            cdt = cd.as_tuple()
+            pdt = pd.as_tuple()
+            assert cdt.__module__ == pdt.__module__
+
+            p = pickle.dumps(cdt, proto)
+            r = pickle.loads(p)
+            assert isinstance(r, C.DecimalTuple)
+            assert cdt == r
+
+            sys.modules["decimal"] = C
+            p = pickle.dumps(cd, proto)
+            sys.modules["decimal"] = P
+            r = pickle.loads(p)
+            assert isinstance(r, P.Decimal)
+            assert r == pd
+
+            sys.modules["decimal"] = C
+            p = pickle.dumps(cdt, proto)
+            sys.modules["decimal"] = P
+            r = pickle.loads(p)
+            assert isinstance(r, P.DecimalTuple)
+            assert r == pdt
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -1,5 +1,9 @@
 # Implementation of the "decimal" module, based on libmpdec library.
 
+__xname__ = __name__    # sys.modules lookup (--without-threads)
+__name__ = 'decimal'    # For pickling
+
+
 import collections as _collections
 import math as _math
 import numbers as _numbers
@@ -23,15 +27,13 @@
 # Errors
 
 class DecimalException(ArithmeticError):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         pass
 
 class Clamped(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class InvalidOperation(DecimalException):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         if args:
             ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True)
@@ -39,41 +41,35 @@
         return _NaN
 
 class ConversionSyntax(InvalidOperation):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class DivisionByZero(DecimalException, ZeroDivisionError):
-    __module__ = 'decimal'
     def handle(self, context, sign, *args):
         return _SignedInfinity[sign]
 
 class DivisionImpossible(InvalidOperation):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class DivisionUndefined(InvalidOperation, ZeroDivisionError):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class Inexact(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class InvalidContext(InvalidOperation):
-    __module__ = 'decimal'
     def handle(self, context, *args):
         return _NaN
 
 class Rounded(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class Subnormal(DecimalException):
-    __module__ = 'decimal'
+    pass
 
 class Overflow(Inexact, Rounded):
-    __module__ = 'decimal'
     def handle(self, context, sign, *args):
         if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN,
                                 ROUND_HALF_DOWN, ROUND_UP):
@@ -90,10 +86,10 @@
                              context.Emax-context.prec+1)
 
 class Underflow(Inexact, Rounded, Subnormal):
-    __module__ = 'decimal'
+    pass
 
 class FloatOperation(DecimalException, TypeError):
-    __module__ = 'decimal'
+    pass
 
 
 __version__ = "1.70"
@@ -107,7 +103,7 @@
 
 def getcontext():
     """Returns this thread's context.
-    
+
     If this thread does not yet have a context, returns
     a new context and sets this thread's context.
     New contexts are copies of DefaultContext.
@@ -173,8 +169,6 @@
 _DEC_MINALLOC = 4
 
 class Decimal(object):
-    __module__ = 'decimal'
-
     __slots__ = ('_mpd', '_data')
 
     def __new__(cls, value="0", context=None):
@@ -326,7 +320,7 @@
             builder.append(b'E')
             builder.append(str(exponent).encode())
 
-        return cls._from_bytes(b''.join(builder), context, exact=exact) 
+        return cls._from_bytes(b''.join(builder), context, exact=exact)
 
     @classmethod
     def from_float(cls, value):
@@ -481,7 +475,7 @@
             numerator = Decimal._from_int(other.numerator, context)
             if not _mpdec.mpd_isspecial(self._mpd):
                 # multiplied = self * other.denominator
-                # 
+                #
                 # Prevent Overflow in the following multiplication.
                 # The result of the multiplication is
                 # only used in mpd_qcmp, which can handle values that
@@ -542,7 +536,7 @@
         _mpdec.mpd_qset_ssize(p._mpd, self._PyHASH_MODULUS,
                               maxctx, status_ptr)
         ten = self._new_empty()
-        _mpdec.mpd_qset_ssize(ten._mpd, 10, 
+        _mpdec.mpd_qset_ssize(ten._mpd, 10,
                               maxctx, status_ptr)
         inv10_p = self._new_empty()
         _mpdec.mpd_qset_ssize(inv10_p._mpd, self._PyHASH_10INV,
@@ -755,7 +749,7 @@
     number_class = _make_unary_operation('number_class')
 
     to_eng_string = _make_unary_operation('to_eng_string')
-    
+
     def fma(self, other, third, context=None):
         context = _getcontext(context)
         return context.fma(self, other, third)
@@ -790,7 +784,7 @@
             result = int.from_bytes(s, 'little', signed=False)
         if _mpdec.mpd_isnegative(x._mpd) and not _mpdec.mpd_iszero(x._mpd):
             result = -result
-        return result        
+        return result
 
     def __int__(self):
         return self._to_int(_mpdec.MPD_ROUND_DOWN)
@@ -798,10 +792,10 @@
     __trunc__ = __int__
 
     def __floor__(self):
-        return self._to_int(_mpdec.MPD_ROUND_FLOOR)        
+        return self._to_int(_mpdec.MPD_ROUND_FLOOR)
 
     def __ceil__(self):
-        return self._to_int(_mpdec.MPD_ROUND_CEILING)        
+        return self._to_int(_mpdec.MPD_ROUND_CEILING)
 
     def to_integral(self, rounding=None, context=None):
         context = _getcontext(context)
@@ -817,7 +811,7 @@
         return result
 
     to_integral_value = to_integral
-        
+
     def to_integral_exact(self, rounding=None, context=None):
         context = _getcontext(context)
         workctx = context.copy()
@@ -886,7 +880,7 @@
         if _mpdec.mpd_isspecial(self._mpd):
             return 0
         return _mpdec.mpd_adjexp(self._mpd)
-    
+
     @property
     def real(self):
         return self
@@ -916,7 +910,7 @@
         fmt = specifier.encode('utf-8')
         context = getcontext()
 
-        replace_fillchar = False 
+        replace_fillchar = False
         if fmt and fmt[0] == 0:
             # NUL fill character: must be replaced with a valid UTF-8 char
             # before calling mpd_parse_fmt_str().
@@ -975,7 +969,7 @@
             result = result.replace(b'\xff', b'\0')
         return result.decode('utf-8')
 
-        
+
 # Register Decimal as a kind of Number (an abstract base class).
 # However, do not register it as Real (because Decimals are not
 # interoperable with floats).
@@ -988,7 +982,7 @@
 
 # Rounding
 _ROUNDINGS = {
-    'ROUND_DOWN': _mpdec.MPD_ROUND_DOWN, 
+    'ROUND_DOWN': _mpdec.MPD_ROUND_DOWN,
     'ROUND_HALF_UP': _mpdec.MPD_ROUND_HALF_UP,
     'ROUND_HALF_EVEN': _mpdec.MPD_ROUND_HALF_EVEN,
     'ROUND_CEILING': _mpdec.MPD_ROUND_CEILING,
@@ -1047,8 +1041,6 @@
     clamp -  If 1, change exponents if too high (Default 0)
     """
 
-    __module__ = 'decimal'
-
     __slots__ = ('_ctx', '_capitals')
 
     def __new__(cls, prec=None, rounding=None, Emin=None, Emax=None,
@@ -1068,7 +1060,7 @@
         ctx.round = _mpdec.MPD_ROUND_HALF_EVEN
         ctx.clamp = 0
         ctx.allcr = 1
-        
+
         self._capitals = 1
         return self
 
@@ -1291,7 +1283,7 @@
         if b is NotImplemented:
             return b, b
         return a, b
-        
+
     def _make_unary_method(name, mpd_func_name):
         mpd_func = getattr(_mpdec, mpd_func_name)
 
@@ -1570,7 +1562,7 @@
 
     def copy(self):
         return self._as_dict()
-        
+
     def __len__(self):
         return len(_SIGNALS)
 
@@ -1629,7 +1621,7 @@
     def __enter__(self):
         self.status_ptr = _ffi.new("uint32_t*")
         return self.context._ctx, self.status_ptr
-        
+
     def __exit__(self, *args):
         status = self.status_ptr[0]
         # May raise a DecimalException

From pypy.commits at gmail.com  Tue Feb 28 10:16:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 07:16:12 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe-2: (fijal, remi, arigo) add a lock
Message-ID: <58b5943c.8c2b190a.1f257.2576@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r90426:41abf85d7a14
Date: 2017-02-28 15:49 +0100
http://bitbucket.org/pypy/pypy/changeset/41abf85d7a14/

Log:	(fijal, remi, arigo) add a lock

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -438,6 +438,12 @@
         self.old_objects_pointing_to_pinned = self.AddressStack()
         self.updated_old_objects_pointing_to_pinned = False
         #
+        # Allocate lock(s)
+        ll_lock = lltype.malloc(rthread.TLOCKP.TO, flavor='raw',
+                                track_allocation=False)
+        rthread.c_thread_lock_init(ll_lock)
+        self.ll_lock = ll_lock
+        #
         # Allocate a nursery.  In case of auto_nursery_size, start by
         # allocating a very small nursery, enough to do things like look
         # up the env var, which requires the GC; and then really
@@ -838,6 +844,8 @@
         major collection, and finally reserve totalsize bytes.
         """
 
+        rthread.acquire_NOAUTO(self.ll_lock, 1)
+
         minor_collection_count = 0
         while True:
             self.set_nursery_free(llmemory.NULL)      # debug: don't use me
@@ -911,6 +919,7 @@
             if self.nursery_top - self.nursery_free > self.debug_tiny_nursery:
                 self.nursery_free = self.nursery_top - self.debug_tiny_nursery
         #
+        rthread.release_NOAUTO(self.ll_lock)
         return result
     collect_and_reserve._dont_inline_ = True
 

From pypy.commits at gmail.com  Tue Feb 28 10:16:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 07:16:15 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe-2: (remi,
 arigo) port some locks from stmgc
Message-ID: <58b5943f.4f072e0a.ad3f0.25c7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r90427:52bbdf98f118
Date: 2017-02-28 16:15 +0100
http://bitbucket.org/pypy/pypy/changeset/52bbdf98f118/

Log:	(remi, arigo) port some locks from stmgc

diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c
--- a/rpython/translator/c/src/mem.c
+++ b/rpython/translator/c/src/mem.c
@@ -15,15 +15,19 @@
 
 static struct pypy_debug_alloc_s *pypy_debug_alloc_list = NULL;
 
+static rpy_spinlock_t pypy_debug_alloc_lock = 0;
+
 RPY_EXTERN
 void pypy_debug_alloc_start(void *addr, const char *funcname)
 {
   struct pypy_debug_alloc_s *p = malloc(sizeof(struct pypy_debug_alloc_s));
   RPyAssert(p, "out of memory");
-  p->next = pypy_debug_alloc_list;
   p->addr = addr;
   p->funcname = funcname;
+  rpy_spinlock_acquire(&pypy_debug_alloc_lock);
+  p->next = pypy_debug_alloc_list;
   pypy_debug_alloc_list = p;
+  rpy_spinlock_release(&pypy_debug_alloc_lock);
 }
 
 RPY_EXTERN
@@ -32,12 +36,14 @@
   struct pypy_debug_alloc_s **p;
   if (!addr)
 	return;
+  rpy_spinlock_acquire(&pypy_debug_alloc_lock);
   for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next))
     if ((*p)->addr == addr)
       {
         struct pypy_debug_alloc_s *dying;
         dying = *p;
         *p = dying->next;
+        rpy_spinlock_release(&pypy_debug_alloc_lock);
         free(dying);
         return;
       }
@@ -49,6 +55,7 @@
 {
   long count = 0;
   struct pypy_debug_alloc_s *p;
+  rpy_spinlock_acquire(&pypy_debug_alloc_lock);
   for (p = pypy_debug_alloc_list; p; p = p->next)
     count++;
   if (count > 0)
@@ -64,6 +71,7 @@
       else
         fprintf(stderr, " (use PYPY_ALLOC=1 to see the list)\n");
     }
+  rpy_spinlock_release(&pypy_debug_alloc_lock);
 }
 
 #endif /* RPY_ASSERT */
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -58,4 +58,15 @@
 //    return &rpy_fastgil;
 }
 
+typedef unsigned char rpy_spinlock_t;
+static inline void rpy_spinlock_acquire(rpy_spinlock_t *p)
+{
+    while (pypy_lock_test_and_set(p, 1) != 0)
+        pypy_spin_loop();
+}
+static inline void rpy_spinlock_release(rpy_spinlock_t *p)
+{
+    pypy_lock_release(p);
+}
+
 #endif
diff --git a/rpython/translator/c/src/thread_pthread.h b/rpython/translator/c/src/thread_pthread.h
--- a/rpython/translator/c/src/thread_pthread.h
+++ b/rpython/translator/c/src/thread_pthread.h
@@ -84,3 +84,9 @@
 
 #define pypy_lock_test_and_set(ptr, value)  __sync_lock_test_and_set(ptr, value)
 #define pypy_lock_release(ptr)              __sync_lock_release(ptr)
+
+#if defined(__i386__) || defined(__amd64__)
+   static inline void pypy_spin_loop(void) { asm("pause" : : : "memory"); }
+#else
+#  define pypy_spin_loop()  /* nothing */
+#endif
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1431,8 +1431,15 @@
         import time, gc
         from rpython.rlib import rthread, rposix
 
+        class X:
+            def __init__(self, prev):
+                self.prev = prev
+
         def bootstrap():
             rthread.gc_thread_start()
+            x = None
+            for i in range(1000):
+                x = X(x)
             os.write(1, "hi there\n")
             rthread.gc_thread_die()
 

From pypy.commits at gmail.com  Tue Feb 28 10:41:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 07:41:54 -0800 (PST)
Subject: [pypy-commit] pypy default: (remi, arigo)
Message-ID: <58b59a42.0b202e0a.e47fc.26bc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r90428:71318996f4ec
Date: 2017-02-28 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/71318996f4ec/

Log:	(remi, arigo)

	- use incminimark in test_standalone, instead of semispace gc

	- fix potential future bug

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1138,7 +1138,7 @@
         # Check if the object at 'addr' is young.
         if not self.is_valid_gc_object(addr):
             return False     # filter out tagged pointers explicitly.
-        if self.nursery <= addr < self.nursery_top:
+        if self.is_in_nursery(addr):
             return True      # addr is in the nursery
         # Else, it may be in the set 'young_rawmalloced_objects'
         return (bool(self.young_rawmalloced_objects) and
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1123,7 +1123,7 @@
 
     def compile(self, entry_point, no__thread=True):
         t = TranslationContext(self.config)
-        t.config.translation.gc = "semispace"
+        t.config.translation.gc = "incminimark"
         t.config.translation.gcrootfinder = self.gcrootfinder
         t.config.translation.thread = True
         t.config.translation.no__thread = no__thread

From pypy.commits at gmail.com  Tue Feb 28 11:23:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Feb 2017 08:23:16 -0800 (PST)
Subject: [pypy-commit] pypy nogil-unsafe-2: (remi, arigo)  general process,
 still completely crashing
Message-ID: <58b5a3f4.e5d7190a.eaad7.2883@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r90429:7be972407621
Date: 2017-02-28 17:22 +0100
http://bitbucket.org/pypy/pypy/changeset/7be972407621/

Log:	(remi, arigo) general process, still completely crashing

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -506,7 +506,7 @@
             if gc_increment_step > 0:
                 self.gc_increment_step = gc_increment_step
             else:
-                self.gc_increment_step = newsize * 4
+                self.gc_increment_step = r_uint(newsize) * 4
             #
             nursery_debug = env.read_uint_from_env('PYPY_GC_NURSERY_DEBUG')
             if nursery_debug > 0:
@@ -514,7 +514,6 @@
             else:
                 self.gc_nursery_debug = False
             self._minor_collection()    # to empty the nursery
-            llarena.arena_free(self.nursery)
             self.nursery_size = newsize
             self.allocate_nursery()
         #
@@ -556,6 +555,17 @@
     def allocate_nursery(self):
         debug_start("gc-set-nursery-size")
         debug_print("nursery size:", self.nursery_size)
+        #
+        if self.nursery:    # free the old nursery
+            llarena.arena_free(self.nursery)
+            self.nursery = llmemory.NULL
+            ll_assert(not self.surviving_pinned_objects.non_empty(),
+                      "pinned objects too early in the process")
+            self.nursery_barriers.delete()
+            self.nursery_barriers = self.AddressDeque()
+            self.set_nursery_free(llmemory.NULL)
+            self.set_nursery_top(llmemory.NULL)
+        #
         self.nursery = self._alloc_nursery()
         # initialize the threshold
         self.min_heap_size = max(self.min_heap_size, self.nursery_size *
@@ -645,10 +655,10 @@
 
     @property
     def nursery_top(self):
-        raise AssertionError, "fix caller"
+        XXX   # fix caller
     @property
     def nursery_free(self):
-        raise AssertionError, "fix caller"
+        XXX   # fix caller
 
 
     def malloc_fixedsize(self, typeid, size,
@@ -752,10 +762,15 @@
             #
             # Get the memory from the nursery.  If there is not enough space
             # there, do a collect first.
-            result = self.nursery_free
-            ll_assert(result != llmemory.NULL, "uninitialized nursery")
-            self.nursery_free = new_free = result + totalsize
-            if new_free > self.nursery_top:
+            result = self.get_nursery_free()
+            if not we_are_translated() and result == llmemory.NULL:
+                # can't do arithmetic from NULL when non-translated
+                grab_next_block = True
+            else:
+                new_free = result + totalsize
+                self.set_nursery_free(new_free)
+                grab_next_block = new_free > self.get_nursery_top()
+            if grab_next_block:
                 result = self.collect_and_reserve(totalsize)
             #
             # Build the object.
@@ -916,8 +931,10 @@
             #
         #
         if self.debug_tiny_nursery >= 0:   # for debugging
-            if self.nursery_top - self.nursery_free > self.debug_tiny_nursery:
-                self.nursery_free = self.nursery_top - self.debug_tiny_nursery
+            if (self.get_nursery_top() - self.get_nursery_free() >
+                                                   self.debug_tiny_nursery):
+                self.set_nursery_free(self.get_nursery_top() -
+                                      self.debug_tiny_nursery)
         #
         rthread.release_NOAUTO(self.ll_lock)
         return result
@@ -1078,7 +1095,7 @@
         if self.next_major_collection_threshold < 0:
             # cannot trigger a full collection now, but we can ensure
             # that one will occur very soon
-            self.nursery_free = self.nursery_top
+            self.set_nursery_free(self.get_nursery_top())
 
     def can_optimize_clean_setarrayitems(self):
         if self.card_page_indices > 0:
@@ -1185,7 +1202,7 @@
         # Check if the object at 'addr' is young.
         if not self.is_valid_gc_object(addr):
             return False     # filter out tagged pointers explicitly.
-        if self.nursery <= addr < self.nursery_top:
+        if self.is_in_nursery(addr):
             return True      # addr is in the nursery
         # Else, it may be in the set 'young_rawmalloced_objects'
         return (bool(self.young_rawmalloced_objects) and
@@ -1675,8 +1692,8 @@
         # nursery_top
         if we_are_translated():
             def zero_nursery_pointers(arg, tl):
-                (tl + NURSERY_FREE.offset).address[0] = llmemory.NULL
-                (tl + NURSERY_TOP.offset).address[0] = llmemory.NULL
+                (tl + NURSERY_FREE._offset).address[0] = llmemory.NULL
+                (tl + NURSERY_TOP._offset).address[0] = llmemory.NULL
             rthread.enum_all_threadlocals(zero_nursery_pointers, None)
         else:
             self.zero_nursery_pointers_in_all_threads()
@@ -2299,7 +2316,7 @@
         debug_print("starting gc state: ", GC_STATES[self.gc_state])
         # Debugging checks
         if self.pinned_objects_in_nursery == 0:
-            ll_assert(self.nursery_free == self.nursery,
+            ll_assert(self.get_nursery_free() == llmemory.NULL,
                       "nursery not empty in major_collection_step()")
         else:
             # XXX try to add some similar check to the above one for the case
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -214,7 +214,7 @@
 
 /* in this situation, we always have one full 'struct pypy_threadlocal_s'
    available, managed by gcc. */
-__thread struct pypy_threadlocal_s pypy_threadlocal;
+__thread struct pypy_threadlocal_s pypy_threadlocal = { 0 };
 
 char *_RPython_ThreadLocals_Build(void)
 {
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1123,7 +1123,7 @@
 
     def compile(self, entry_point, no__thread=True):
         t = TranslationContext(self.config)
-        t.config.translation.gc = "semispace"
+        t.config.translation.gc = "incminimark"
         t.config.translation.gcrootfinder = self.gcrootfinder
         t.config.translation.thread = True
         t.config.translation.no__thread = no__thread
@@ -1438,7 +1438,7 @@
         def bootstrap():
             rthread.gc_thread_start()
             x = None
-            for i in range(1000):
+            for i in range(1000000):
                 x = X(x)
             os.write(1, "hi there\n")
             rthread.gc_thread_die()

From pypy.commits at gmail.com  Tue Feb 28 13:09:37 2017
From: pypy.commits at gmail.com (robert-zaremba)
Date: Tue, 28 Feb 2017 10:09:37 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix logging.getLevelName,
 applying #29220 CPython issue.
Message-ID: <58b5bce1.690f190a.9a200.2cce@mx.google.com>

Author: Robert Zaremba <robert.zaremba at scale-it.pl>
Branch: py3.5
Changeset: r90430:55230b1507ef
Date: 2017-02-28 18:33 +0100
http://bitbucket.org/pypy/pypy/changeset/55230b1507ef/

Log:	Fix logging.getLevelName, applying #29220 CPython issue.

	The getLevelName function incorrectly fallsback. 0 is correct value
	returned from the names map, but when using `or` check it falsifies
	and fallback to the "level" case.

	http://buildbot.pypy.org/summary/longrepr?testname=unmodified&builde
	r=pypy-c-jit-linux-x86-64&build=4416&mod=lib-
	python.3.test.test_logging

diff --git a/lib-python/3/logging/__init__.py b/lib-python/3/logging/__init__.py
--- a/lib-python/3/logging/__init__.py
+++ b/lib-python/3/logging/__init__.py
@@ -129,9 +129,14 @@
 
     Otherwise, the string "Level %s" % level is returned.
     """
-    # See Issues #22386 and #27937 for why it's this way
-    return (_levelToName.get(level) or _nameToLevel.get(level) or
-            "Level %s" % level)
+    # See Issues #22386, #27937 and #29220 for why it's this way
+    result = _levelToName.get(level)
+    if result is not None:
+        return result
+    result = _nameToLevel.get(level)
+    if result is not None:
+        return result
+    return "Level %s" % level
 
 def addLevelName(level, levelName):
     """

From pypy.commits at gmail.com  Tue Feb 28 17:11:04 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 28 Feb 2017 14:11:04 -0800 (PST)
Subject: [pypy-commit] pypy default: hack hack hack getnightly.py until it
 extracts also the *.h files in the proper dir
Message-ID: <58b5f578.c796190a.34c78.36ab@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r90431:e2e364614275
Date: 2017-02-27 12:06 +0100
http://bitbucket.org/pypy/pypy/changeset/e2e364614275/

Log:	hack hack hack getnightly.py until it extracts also the *.h files in
	the proper dir

diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py
--- a/pypy/goal/getnightly.py
+++ b/pypy/goal/getnightly.py
@@ -4,16 +4,24 @@
 import os
 import py
 
+TAR_OPTIONS = '-x -v --strip-components=2'
+TAR = 'tar {options} -f {tarfile} {files}'
+
+def untar(tarfile, files):
+    cmd = TAR.format(options=TAR_OPTIONS, tarfile=tarfile, files=files)
+    os.system(cmd)
+
 if sys.platform.startswith('linux'):
     arch = 'linux'
     cmd = 'wget "%s"'
-    tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'"
+    TAR_OPTIONS += ' --wildcards'
+    binfiles = "'*/bin/pypy' '*/bin/libpypy-c.so'"
     if os.uname()[-1].startswith('arm'):
         arch += '-armhf-raspbian'
 elif sys.platform.startswith('darwin'):
     arch = 'osx'
     cmd = 'curl -O "%s"'
-    tar = "tar -x -v --strip-components=2 -f %s '*/bin/pypy'"
+    binfiles = "'*/bin/pypy'"
 else:
     print 'Cannot determine the platform, please update this script'
     sys.exit(1)
@@ -34,6 +42,7 @@
 filename = 'pypy-c-%s-latest-%s.tar.bz2' % (kind, arch)
 url = 'http://buildbot.pypy.org/nightly/%s/%s' % (branch, filename)
 tmp = py.path.local.mkdtemp()
+pypy_latest = tmp.join(filename)
 mydir = tmp.chdir()
 print 'Downloading pypy to', tmp
 if os.system(cmd % url) != 0:
@@ -41,4 +50,10 @@
 
 print 'Extracting pypy binary'
 mydir.chdir()
-os.system(tar % tmp.join(filename))
+untar(pypy_latest, binfiles)
+include_dir = py.path.local('../../include')
+if include_dir.check(dir=True):
+    include_dir.chdir()
+    untar(pypy_latest, '*/include/*')
+else:
+    print 'WARNING: could not find the include/ dir'

From pypy.commits at gmail.com  Tue Feb 28 17:11:09 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 28 Feb 2017 14:11:09 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <58b5f57d.55582e0a.da759.3597@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r90433:fdae6b92de2f
Date: 2017-02-28 23:10 +0100
http://bitbucket.org/pypy/pypy/changeset/fdae6b92de2f/

Log:	merge heads

diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py
--- a/pypy/goal/getnightly.py
+++ b/pypy/goal/getnightly.py
@@ -4,16 +4,24 @@
 import os
 import py
 
+TAR_OPTIONS = '-x -v --strip-components=2'
+TAR = 'tar {options} -f {tarfile} {files}'
+
+def untar(tarfile, files):
+    cmd = TAR.format(options=TAR_OPTIONS, tarfile=tarfile, files=files)
+    os.system(cmd)
+
 if sys.platform.startswith('linux'):
     arch = 'linux'
     cmd = 'wget "%s"'
-    tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'"
+    TAR_OPTIONS += ' --wildcards'
+    binfiles = "'*/bin/pypy' '*/bin/libpypy-c.so'"
     if os.uname()[-1].startswith('arm'):
         arch += '-armhf-raspbian'
 elif sys.platform.startswith('darwin'):
     arch = 'osx'
     cmd = 'curl -O "%s"'
-    tar = "tar -x -v --strip-components=2 -f %s '*/bin/pypy'"
+    binfiles = "'*/bin/pypy'"
 else:
     print 'Cannot determine the platform, please update this script'
     sys.exit(1)
@@ -34,6 +42,7 @@
 filename = 'pypy-c-%s-latest-%s.tar.bz2' % (kind, arch)
 url = 'http://buildbot.pypy.org/nightly/%s/%s' % (branch, filename)
 tmp = py.path.local.mkdtemp()
+pypy_latest = tmp.join(filename)
 mydir = tmp.chdir()
 print 'Downloading pypy to', tmp
 if os.system(cmd % url) != 0:
@@ -41,4 +50,10 @@
 
 print 'Extracting pypy binary'
 mydir.chdir()
-os.system(tar % tmp.join(filename))
+untar(pypy_latest, binfiles)
+include_dir = py.path.local('../../include')
+if include_dir.check(dir=True):
+    include_dir.chdir()
+    untar(pypy_latest, '*/include/*')
+else:
+    print 'WARNING: could not find the include/ dir'

From pypy.commits at gmail.com  Tue Feb 28 17:11:06 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 28 Feb 2017 14:11:06 -0800 (PST)
Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: (antocuni,
 rlamy): modify an existing test to show broken behavior: the
 correct way to implement bf_getbuffer is to do a Py_INCREF(obj),
 which the broken test didn't. Once you do,
 you notice that cpyext never does the corresponding decref,
 thus causing a leak because the original object is never deallocated
Message-ID: <58b5f57a.1ae9190a.96572.3878@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: fix-cpyext-releasebuffer
Changeset: r90432:0d60d6a74fdc
Date: 2017-02-28 23:09 +0100
http://bitbucket.org/pypy/pypy/changeset/0d60d6a74fdc/

Log:	(antocuni, rlamy): modify an existing test to show broken behavior:
	the correct way to implement bf_getbuffer is to do a Py_INCREF(obj),
	which the broken test didn't. Once you do, you notice that cpyext
	never does the corresponding decref, thus causing a leak because the
	original object is never deallocated

diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -73,16 +73,27 @@
                 return obj;
              """),
             ("get_cnt", "METH_NOARGS",
-             'return PyLong_FromLong(cnt);')], prologue="""
+             'return PyLong_FromLong(cnt);'),
+            ("get_dealloc_cnt", "METH_NOARGS",
+             'return PyLong_FromLong(dealloc_cnt);'),
+        ],
+        prologue="""
                 static float test_data = 42.f;
                 static int cnt=0;
+                static int dealloc_cnt=0;
                 static PyHeapTypeObject * type=NULL;
 
+                void dealloc(PyObject *self) {
+                    dealloc_cnt++;
+                }
                 int getbuffer(PyObject *obj, Py_buffer *view, int flags) {
 
                     cnt ++;
                     memset(view, 0, sizeof(Py_buffer));
                     view->obj = obj;
+                    /* see the CPython docs for why we need this incref:
+                       https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer */
+                    Py_INCREF(obj);
                     view->ndim = 0;
                     view->buf = (void *) &test_data;
                     view->itemsize = sizeof(float);
@@ -96,7 +107,7 @@
                 void releasebuffer(PyObject *obj, Py_buffer *view) { 
                     cnt --;
                 }
-            """, more_init="""
+            """, more_init="""            
                 type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
 
                 type->ht_type.tp_name = "Test";
@@ -106,6 +117,7 @@
                                           Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_NEWBUFFER;
                 type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC;
 
+                type->ht_type.tp_dealloc = dealloc;
                 type->ht_type.tp_as_buffer = &type->as_buffer;
                 type->as_buffer.bf_getbuffer = getbuffer;
                 type->as_buffer.bf_releasebuffer = releasebuffer;
@@ -116,6 +128,8 @@
         assert module.get_cnt() == 0
         a = memoryview(module.create_test())
         assert module.get_cnt() == 1
+        assert module.get_dealloc_cnt() == 0
         del a
-        gc.collect(); gc.collect(); gc.collect()
+        self.debug_collect()
         assert module.get_cnt() == 0
+        assert module.get_dealloc_cnt() == 1